diff --git a/.gitignore b/.gitignore index ab7b73c9..e0757be1 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,4 @@ eaConf.json exotic.log.* pl_names.json /.project +/.deps313 diff --git a/README.md b/README.md index 49a6c7f2..f0cdd235 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,7 @@ Get EXOTIC up and running faster with a json file. Please see the included file "AAVSO Observer Code (blank if none)": "RTZ", "Secondary Observer Codes (blank if none)": "", + "Observatory Full Title": "", "Observation date": "17-December-2017", "Obs. Latitude": "+32.41638889", @@ -120,10 +121,11 @@ Get EXOTIC up and running faster with a json file. Please see the included file "Filter Name (aavso.org/filters)": "V", "Observing Notes": "Weather, seeing was nice.", - "Plate Solution? (y/n)": "y", + "Plate Solution? (y/n)": true, "Target Star X & Y Pixel": [424, 286], - "Comparison Star(s) X & Y Pixel": [[465, 183], [512, 263], [], [], [], [], [], [], [], []] + "Comparison Star(s) X & Y Pixel": [[465, 183], [512, 263], [], [], [], [], [], [], [], []], + "Comparison Star(s) RA & Dec": null }, "planetary_parameters": { "Target Star RA": "02:04:10", @@ -159,6 +161,32 @@ Get EXOTIC up and running faster with a json file. Please see the included file "Filter Minimum Wavelength (nm)": null, "Filter Maximum Wavelength (nm)": null, + "Fast Aperture Mask (y/n)": false, + "allow_pixel_alignment_fallback": true, + "prefer_pixel_values_over_wcs_for_target": false, + "use_psf_photometry": true, + "use_aperture_photometry": true, + "use_aperture_corrections_and_full_image_fwhm": false, + "use_ensemble_photometry_rather_than_single_comp": false, + "stellar_variability_only": false, + "use_ensemble_photometry_for_stellar_variability": true, + "require_apparent_magnitudes": true, + "use_exactly_the_comps_provided": false, + "maximum_number_of_ensemble_comparisons_for_transit": 5, + "maximum_number_of_ensemble_comparisons_for_stellar_variability": 5, + "photometer_fortuitous_variables": true, + "use_single_comparison_for_fortuitous_variables": true, + "use_nextastro_vsx_cache_first": false, + "skip_low_comparison_coverage_rejection": false, + "fit_lightcurve_to_every_comparison_candidate": false, + "detrend_on_outoftransit_baseline": true, + "final_fit_baseline_duration_multiplier": 1.0, + "use_eebls_to_initialize_tmid_and_bounds": true, + "pick_comparison_by_eebls_snr": true, + "use_impactparameter_rather_than_inclination_to_fit": true, + "Use target-driven comp selection rather than comp-driven comp selection": false, + "require_comp_star": true, + "Pixel Scale (Ex: 5.21 arcsecs/pixel)": null, "Exposure Time (s)": 60.0 @@ -166,6 +194,38 @@ Get EXOTIC up and running faster with a json file. Please see the included file } ``` +### Comparison-star mode tags + +Put these tags in the top-level `"optional_info"` object. JSON booleans (`true` and `false`) are recommended. Every initialization boolean also accepts numeric `1`/`0` and case-insensitive strings `"y"`/`"n"`, `"yes"`/`"no"`, `"true"`/`"false"`, and `"on"`/`"off"`. + +Raw-image reductions prefer per-frame WCS when WCS coverage is consistent across the dataset. With the default `"allow_pixel_alignment_fallback": true`, EXOTIC uses `"bad_wcs_threshold_percent"` to choose the safe path: sparse missing-WCS frames below the threshold are dropped and the retained sequence remains WCS-based; when the missing-WCS fraction reaches or exceeds the threshold, all frames are retained and legacy pixel alignment is available for frames without usable WCS. Set `"allow_pixel_alignment_fallback": false` to require WCS-only processing and drop every frame without celestial WCS. The existing `"Ignore WCS in Header and Do Manual Alignment? (y/n)": "y"` option explicitly enables pixel alignment for the entire run. + +Comparison stars may be supplied in `user_info` using either `"Comparison Star(s) X & Y Pixel"` or `"Comparison Star(s) RA & Dec"`. Do not populate both. RA/Dec values may be decimal degrees, such as `[[31.04125, 46.68972]]`, or sexagesimal strings, such as `[["02:04:09.90", "+46:41:23.0"]]`. Sexagesimal values must be quoted because they are JSON strings; forms such as `[[02:04:09.90, +46:41:23.0]]` are not valid JSON. Supplied X/Y positions are converted to sky coordinates with the reference frame's WCS; during photometry those sky coordinates are projected independently through every retained frame's own WCS header. + +| Reduction | Requested comparison mode | `optional_info` settings | +|---|---|---| +| Transit fit | Single comparison star (default) | `"stellar_variability_only": false`, `"require_comp_star": true`, `"use_ensemble_photometry_rather_than_single_comp": false` | +| Transit fit | Comparison-star ensemble | `"stellar_variability_only": false`, `"require_comp_star": true`, `"use_ensemble_photometry_rather_than_single_comp": true`, `"maximum_number_of_ensemble_comparisons_for_transit": 5` | +| Transit or variability run | Exactly the supplied comparison(s) | `"use_exactly_the_comps_provided": true`. Comparisons may be supplied as X/Y or RA/Dec. One supplied comparison is used alone; two or more are all used as one fixed ensemble. Automatic replacement, addition, VSX/stability vetting, ranking, and ensemble-size limiting are bypassed. | +| Transit fit | No comparison star | There is no tag that forces this mode. `"require_comp_star": false` only removes the requirement for a comparison star; it does not force target-only photometry. The current comparison-calibration FITS path still selects a single comparison or an ensemble. | +| Stellar-variability-only run | Single comparison star | `"stellar_variability_only": true`, `"use_ensemble_photometry_for_stellar_variability": false` | +| Stellar-variability-only run | Calibrated comparison-star ensemble (default) | `"stellar_variability_only": true`, `"use_ensemble_photometry_for_stellar_variability": true`, `"maximum_number_of_ensemble_comparisons_for_stellar_variability": 5` | +| Stellar-variability-only run | No comparison star | Not supported for raw-FITS absolute variability photometry; a single calibrated comparison or calibrated ensemble is required. A pre-reduced relative light curve can be supplied without raw comparison-star photometry, but it is not selected by a comparison-mode tag. | + +The two ensemble limits are independent. `"maximum_number_of_ensemble_comparisons_for_transit"` caps only the transit-fit ensemble. `"maximum_number_of_ensemble_comparisons_for_stellar_variability"` caps both stellar-variability-only and fortuitous-variable ensembles. Each defaults to `5`, must be an integer of at least `2`, and has no configured upper limit. Increase either value to permit a much larger ensemble; EXOTIC will enlarge automatic candidate discovery for the corresponding ensemble where applicable, then use up to that number of surviving comparisons. Very large ensembles require more photometry work. Stellar-variability and fortuitous-variable ensembles can also retain fewer frames because every selected member must have a usable measurement in a retained frame. + +Ensemble settings retain a single-comparison fallback when EXOTIC cannot build a usable ensemble, except when `"use_exactly_the_comps_provided"` is true. Exact-comparison mode fails explicitly if the supplied reference cannot be measured; it never silently substitutes or drops a supplied comparison. This makes the same reference star or ensemble reproducible across multiple runs. + +Differential-magnitude CSV and plot products are always attempted independently of catalogue calibration. Set `"require_apparent_magnitudes": false` when catalogue-calibrated apparent magnitudes are not required; EXOTIC still writes apparent-magnitude products when calibration is available. Stellar-variability apparent and differential magnitudes use the raw target/reference flux ratio and are explicitly not airmass-corrected, because a real time-dependent stellar signal can be correlated with airmass. Airmass remains in the output as metadata. + +Flux-bearing result files retain both magnitude representations. Final-lightcurve and differential-magnitude CSV rows explicitly include the raw, uncorrected differential magnitude and uncertainty alongside the corrected differential magnitude and uncertainty, plus apparent magnitude and uncertainty where applicable (or `na` when no catalogue calibration is available). Transit AAVSO files retain their standard exoplanet columns and add one preserved `#MAGNITUDE-XC` record per data row containing both raw and corrected differential values and the applied correction factor. When weighted linear out-of-transit baseline detrending is applied, `#OUT_OF_TRANSIT_BASELINE-XC` records the formula, BJD_TDB reference time, intercept, and slope; the standard `DIFF` and `ERR` rows are restored to their pre-detrending values and `DETREND_2` carries the correction function, making the operation reversible from the AAVSO file. AID rows retain the standard Extended format and store raw `DIFFMAG` and `DIFFERR` values while `MAG` and `MERR` remain the apparent magnitude measurement. All transit and AID AAVSO files are written in an `AAVSO_Files` subfolder of their corresponding output directory. That folder also receives copies of the final-lightcurve PNG, PDF, and CSV; every FOV finder-chart PNG and PDF; the normal, final, and zoomed triangle plots; the KTMF QC PNG and PDF; and the prior-versus-posterior comparison PNG and PDF. Finder charts label every selected comparison member, including ensembles and comparisons sourced outside AAVSO. This preserves the target-minus-reference measurement needed to apply a revised apparent-magnitude calibration later. Each reduction writes its run log from startup through shutdown to a unique `Diagnostics/EXOTIC_RunLog__pid.log`, so separate or midnight-spanning runs do not overwrite or split one another. + +For fortuitous VSX variables found during a transit reduction, `"photometer_fortuitous_variables": true` turns their photometry on; `"use_single_comparison_for_fortuitous_variables": true` selects one comparison (the default), while `false` requests an ensemble capped by `"maximum_number_of_ensemble_comparisons_for_stellar_variability"`. Fortuitous-variable differential products remain available when catalogue calibration is unavailable. Fortuitous-variable photometry has no no-comparison mode. + +`photometer_fortuitous_variables` defaults to `true` for full FITS reductions with a WCS. EXOTIC searches the field in VSX, retains unsaturated stars whose reference-image source-plus-sky noise estimate implies an internal error below 0.05 mag, and measures each retained variable against one calibrated comparison star by default, or against its own calibrated comparison ensemble when `"use_single_comparison_for_fortuitous_variables"` is `false`. Exported light curves also retain only frames whose final comparison-calibrated internal magnitude error is below 0.05 mag. Each VSX target uses its own frame-level saturation mask: saturation of the exoplanet target does not remove that image from the VSX target's run, while saturated measurements of that VSX target or a reference star are masked only for the affected source and frame. The ensemble's high-side comparison-catalog error sigma clip has a 0.01 mag minimum threshold, so comparison errors at or below 0.01 mag are never rejected by that clip. Every ensemble AAVSO AID file includes an `#ENSEMBLE-COMPARISONS-XC` JSON header listing every selected comparison star with its label, RA, Dec, pixel position, and catalog calibration. Per-star plots, magnitude CSV, and ensemble-selection JSON are written below `variables/optimal_variables//` when the VSX period is at most 10 days and amplitude is at least 0.3 mag, or below `variables/normal//` otherwise; each AAVSO AID file is placed in that variable directory's `AAVSO_Files` subfolder. Skipped variables are recorded only in the shared `variables/FortuitousVariables_.json` manifest and do not receive an object directory. Set `"photometer_fortuitous_variables"` to `false` to disable these products. + +`use_nextastro_vsx_cache_first` defaults to `false`. When enabled, fortuitous-variable discovery queries `https://photometry.nextastro.org/vsx_query` first. EXOTIC falls back to AAVSO when the cache fails or returns no objects. Full-schema cache responses supply period and amplitude directly; legacy cache responses are enriched from AAVSO for optimal/normal classification. + ## Features and Pipeline Architecture - Automatic Plate Solution from http://nova.astrometry.net diff --git a/docs/README.md b/docs/README.md index 732c61ac..64bcbeba 100644 --- a/docs/README.md +++ b/docs/README.md @@ -109,6 +109,7 @@ The scatter in the residuals of the lightcurve fit is: 0.5414 % - If you do not have any of these calibrations, enter `null` - AAVSO Observer Code - if you do not have one, leave as N/A - Secondary Observer Codes - the AAVSO observer codes of anyone who helped out with your observations; if you do not have one, leave as N/A + - Observatory Full Title - optional full observatory name; if provided, EXOTIC writes it to the AAVSO header as `OBSNAME` - Observation date - the date of your observation in DAY-MONTH-YEAR format - Obs. Latitude - the latitude of your observations, where North is denoted with a + and South is denoted with a - - Obs. Longitude - the longitude of your observations, where East is denoted with a + and West is denoted with a - @@ -121,7 +122,7 @@ The scatter in the residuals of the lightcurve fit is: 0.5414 % - Plate solve my images - select if you want EXOTIC to calibrate the right ascenscion and declination of your pixels in your image via Astrometry.net; it is recommended that this option is selected - Align my images - select this option for EXOTIC to align all of your images to provide better tracking of your stars in your images; it is recommended that this option is selected - Target Star X & Y Pixel Position - the pixel location of your target exoplanet host star in [x-position, y-position] format - - Comparison Star(s) X & Y Pixel Position - the pixel location of your comparision star(s) in [x-position, y-position] format; it is recommended that you input at least 2 comparision stars and EXOTIC will automatically select the "best" comparision by the one that produces the least amount of scatter in your data + - Comparison Star(s) Position - provide either X/Y pixel pairs or RA/Dec pairs, but not both. RA/Dec accepts decimal degrees or sexagesimal strings, requires a usable WCS, and is projected onto the selected reference image before being treated exactly like X/Y input - *NOTE:* In the screenshot below, Rob has already entered all of the information for you for the sample data (with the exception that you'll need to point to the correct directory for your FITS files and your EXOTIC Output) ![EXOTIC Input Observation Information](https://github.com/rzellem/EXOTIC/blob/develop/docs/images/exotic_inputobs.png) @@ -167,6 +168,7 @@ Get EXOTIC up and running faster with a json file. Please see the included file "AAVSO Observer Code (N/A if none)": "RTZ", "Secondary Observer Codes (N/A if none)": "N/A", + "Observatory Full Title": "", "Observation date": "December 17, 2017", "Obs. Latitude": "+31.68", @@ -177,10 +179,11 @@ Get EXOTIC up and running faster with a json file. Please see the included file "Filter Name (aavso.org/filters)": "V", "Observing Notes": "Weather, seeing was nice.", - "Plate Solution? (y/n)": "n", + "Plate Solution? (y/n)": false, "Target Star X & Y Pixel": [424, 286], - "Comparison Star(s) X & Y Pixel": [[465, 183], [512, 263]] + "Comparison Star(s) X & Y Pixel": [[465, 183], [512, 263]], + "Comparison Star(s) RA & Dec": null }, "planetary_parameters": { "Target Star RA": "02:04:10", @@ -211,7 +214,61 @@ Get EXOTIC up and running faster with a json file. Please see the included file "optional_info": { "Pixel Scale (Ex: 5.21 arcsecs/pixel)": null, "Filter Minimum Wavelength (nm)": null, - "Filter Maximum Wavelength (nm)": null + "Filter Maximum Wavelength (nm)": null, + "Fast Aperture Mask (y/n)": false, + "allow_pixel_alignment_fallback": true, + "prefer_pixel_values_over_wcs_for_target": false, + "use_psf_photometry": true, + "use_aperture_photometry": true, + "use_aperture_corrections_and_full_image_fwhm": false, + "use_ensemble_photometry_rather_than_single_comp": false, + "stellar_variability_only": false, + "use_ensemble_photometry_for_stellar_variability": true, + "require_apparent_magnitudes": true, + "use_exactly_the_comps_provided": false, + "maximum_number_of_ensemble_comparisons_for_transit": 5, + "maximum_number_of_ensemble_comparisons_for_stellar_variability": 5, + "photometer_fortuitous_variables": true, + "use_single_comparison_for_fortuitous_variables": true, + "use_nextastro_vsx_cache_first": false, + "detrend_on_outoftransit_baseline": true, + "use_impactparameter_rather_than_inclination_to_fit": true, + "skip_low_comparison_coverage_rejection": false, + "require_comp_star": true } } ``` + +### Comparison-star mode tags + +Put these tags in the top-level `"optional_info"` object. JSON booleans (`true` and `false`) are recommended. Every initialization boolean also accepts numeric `1`/`0` and case-insensitive strings `"y"`/`"n"`, `"yes"`/`"no"`, `"true"`/`"false"`, and `"on"`/`"off"`. + +Raw-image reductions prefer per-frame WCS when WCS coverage is consistent across the dataset. With the default `"allow_pixel_alignment_fallback": true`, EXOTIC uses `"bad_wcs_threshold_percent"` to choose the safe path: sparse missing-WCS frames below the threshold are dropped and the retained sequence remains WCS-based; when the missing-WCS fraction reaches or exceeds the threshold, all frames are retained and legacy pixel alignment is available for frames without usable WCS. Set `"allow_pixel_alignment_fallback": false` to require WCS-only processing and drop every frame without celestial WCS. The existing `"Ignore WCS in Header and Do Manual Alignment? (y/n)": "y"` option explicitly enables pixel alignment for the entire run. + +Comparison stars may be supplied in `user_info` using either `"Comparison Star(s) X & Y Pixel"` or `"Comparison Star(s) RA & Dec"`. Do not populate both. Supplied X/Y positions are converted to sky coordinates with the reference frame's WCS; during photometry those sky coordinates are projected independently through every retained frame's own WCS header. + +| Reduction | Requested comparison mode | `optional_info` settings | +|---|---|---| +| Transit fit | Single comparison star (default) | `"stellar_variability_only": false`, `"require_comp_star": true`, `"use_ensemble_photometry_rather_than_single_comp": false` | +| Transit fit | Comparison-star ensemble | `"stellar_variability_only": false`, `"require_comp_star": true`, `"use_ensemble_photometry_rather_than_single_comp": true`, `"maximum_number_of_ensemble_comparisons_for_transit": 5` | +| Transit or variability run | Exactly the supplied comparison(s) | `"use_exactly_the_comps_provided": true`. Comparisons may be supplied as X/Y or RA/Dec. One supplied comparison is used alone; two or more are all used as one fixed ensemble. Automatic replacement, addition, VSX/stability vetting, ranking, and ensemble-size limiting are bypassed. | +| Transit fit | No comparison star | There is no tag that forces this mode. `"require_comp_star": false` only removes the requirement for a comparison star; it does not force target-only photometry. The current comparison-calibration FITS path still selects a single comparison or an ensemble. | +| Stellar-variability-only run | Single comparison star | `"stellar_variability_only": true`, `"use_ensemble_photometry_for_stellar_variability": false` | +| Stellar-variability-only run | Calibrated comparison-star ensemble (default) | `"stellar_variability_only": true`, `"use_ensemble_photometry_for_stellar_variability": true`, `"maximum_number_of_ensemble_comparisons_for_stellar_variability": 5` | +| Stellar-variability-only run | No comparison star | Not supported for raw-FITS absolute variability photometry; a single calibrated comparison or calibrated ensemble is required. A pre-reduced relative light curve can be supplied without raw comparison-star photometry, but it is not selected by a comparison-mode tag. | + +The two ensemble limits are independent. `"maximum_number_of_ensemble_comparisons_for_transit"` caps only the transit-fit ensemble. `"maximum_number_of_ensemble_comparisons_for_stellar_variability"` caps both stellar-variability-only and fortuitous-variable ensembles. Each defaults to `5`, must be an integer of at least `2`, and has no configured upper limit. Increase either value to permit a much larger ensemble; EXOTIC will enlarge automatic candidate discovery for the corresponding ensemble where applicable, then use up to that number of surviving comparisons. Very large ensembles require more photometry work. Stellar-variability and fortuitous-variable ensembles can also retain fewer frames because every selected member must have a usable measurement in a retained frame. + +Ensemble settings retain a single-comparison fallback when EXOTIC cannot build a usable ensemble, except when `"use_exactly_the_comps_provided"` is true. Exact-comparison mode fails explicitly if the supplied reference cannot be measured; it never silently substitutes or drops a supplied comparison. This makes the same reference star or ensemble reproducible across multiple runs. + +Comparison stars may be supplied in `user_info` using either `"Comparison Star(s) X & Y Pixel"` or `"Comparison Star(s) RA & Dec"`. Do not populate both. RA/Dec values may be decimal degrees, such as `[[31.04125, 46.68972]]`, or sexagesimal strings, such as `[["02:04:09.90", "+46:41:23.0"]]`. Sexagesimal values must be quoted because they are JSON strings; forms such as `[[02:04:09.90, +46:41:23.0]]` are not valid JSON. Celestial coordinates require a usable WCS and are projected onto the selected reference image before photometry; after projection they are treated identically to supplied X/Y positions. + +Differential-magnitude CSV and plot products are always attempted independently of catalogue calibration. Set `"require_apparent_magnitudes": false` when catalogue-calibrated apparent magnitudes are not required; EXOTIC still writes apparent-magnitude products when calibration is available. Stellar-variability apparent and differential magnitudes use the raw target/reference flux ratio and are explicitly not airmass-corrected, because a real time-dependent stellar signal can be correlated with airmass. Airmass remains in the output as metadata. + +Flux-bearing result files retain both magnitude representations. Final-lightcurve and differential-magnitude CSV rows explicitly include the raw, uncorrected differential magnitude and uncertainty alongside the corrected differential magnitude and uncertainty, plus apparent magnitude and uncertainty where applicable (or `na` when no catalogue calibration is available). Transit AAVSO files retain their standard exoplanet columns and add one preserved `#MAGNITUDE-XC` record per data row containing both raw and corrected differential values and the applied correction factor. When weighted linear out-of-transit baseline detrending is applied, `#OUT_OF_TRANSIT_BASELINE-XC` records the formula, BJD_TDB reference time, intercept, and slope; the standard `DIFF` and `ERR` rows are restored to their pre-detrending values and `DETREND_2` carries the correction function, making the operation reversible from the AAVSO file. AID rows retain the standard Extended format and store raw `DIFFMAG` and `DIFFERR` values while `MAG` and `MERR` remain the apparent magnitude measurement. All transit and AID AAVSO files are written in an `AAVSO_Files` subfolder of their corresponding output directory. That folder also receives copies of the final-lightcurve PNG, PDF, and CSV; every FOV finder-chart PNG and PDF; the normal, final, and zoomed triangle plots; the KTMF QC PNG and PDF; and the prior-versus-posterior comparison PNG and PDF. Finder charts label every selected comparison member, including ensembles and comparisons sourced outside AAVSO. This preserves the target-minus-reference measurement needed to apply a revised apparent-magnitude calibration later. Each reduction writes its run log from startup through shutdown to a unique `Diagnostics/EXOTIC_RunLog__pid.log`, so separate or midnight-spanning runs do not overwrite or split one another. + +For fortuitous VSX variables found during a transit reduction, `"photometer_fortuitous_variables": true` turns their photometry on; `"use_single_comparison_for_fortuitous_variables": true` selects one comparison (the default), while `false` requests an ensemble capped by `"maximum_number_of_ensemble_comparisons_for_stellar_variability"`. Fortuitous-variable differential products remain available when catalogue calibration is unavailable. Fortuitous-variable photometry has no no-comparison mode. + +`photometer_fortuitous_variables` defaults to `true` for full FITS reductions with a WCS. EXOTIC searches the field in VSX, retains unsaturated stars whose reference-image source-plus-sky noise estimate implies an internal error below 0.05 mag, and measures each retained variable against one calibrated comparison star by default, or against its own calibrated comparison ensemble when `"use_single_comparison_for_fortuitous_variables"` is `false`. Exported light curves also retain only frames whose final comparison-calibrated internal magnitude error is below 0.05 mag. Each VSX target uses its own frame-level saturation mask: saturation of the exoplanet target does not remove that image from the VSX target's run, while saturated measurements of that VSX target or a reference star are masked only for the affected source and frame. The ensemble's high-side comparison-catalog error sigma clip has a 0.01 mag minimum threshold, so comparison errors at or below 0.01 mag are never rejected by that clip. Every ensemble AAVSO AID file includes an `#ENSEMBLE-COMPARISONS-XC` JSON header listing every selected comparison star with its label, RA, Dec, pixel position, and catalog calibration. Per-star plots, magnitude CSV, and ensemble-selection JSON are written below `variables/optimal_variables//` when the VSX period is at most 10 days and amplitude is at least 0.3 mag, or below `variables/normal//` otherwise; each AAVSO AID file is placed in that variable directory's `AAVSO_Files` subfolder. Skipped variables are recorded only in the shared `variables/FortuitousVariables_.json` manifest and do not receive an object directory. Set `"photometer_fortuitous_variables"` to `false` to disable these products. + +`use_nextastro_vsx_cache_first` defaults to `false`. When enabled, fortuitous-variable discovery queries `https://photometry.nextastro.org/vsx_query` first. EXOTIC falls back to AAVSO when the cache fails or returns no objects. Full-schema cache responses supply period and amplitude directly; legacy cache responses are enriched from AAVSO for optimal/normal classification. diff --git a/docs/regions/English/example_output.txt b/docs/regions/English/example_output.txt index 7a5d4f62..5154f9c7 100644 --- a/docs/regions/English/example_output.txt +++ b/docs/regions/English/example_output.txt @@ -227,7 +227,7 @@ The Mean Squared Error is: 255638.858211 ********************************************* -Best Comparison Star: #2 +Transit Fit Comparison Star: #2 Minimum Residual Scatter: 0.5414% Optimal Aperture: 4 Optimal Annulus: 5 @@ -261,4 +261,4 @@ Output File Saved ************************ End of Reduction Process -************************ \ No newline at end of file +************************ diff --git a/docs/regions/German/Beispiel_Output.txt b/docs/regions/German/Beispiel_Output.txt index 2e9be3d0..342ce830 100644 --- a/docs/regions/German/Beispiel_Output.txt +++ b/docs/regions/German/Beispiel_Output.txt @@ -227,7 +227,7 @@ The Mean Squared Error is: 255638.858211 ********************************************* -Best Comparison Star: #2 +Transit Fit Comparison Star: #2 Minimum Residual Scatter: 0.5414% Optimal Aperture: 4 Optimal Annulus: 5 @@ -261,4 +261,4 @@ Output File Saved ************************ End of Reduction Process -************************ \ No newline at end of file +************************ diff --git a/docs/system_prompt.txt b/docs/system_prompt.txt index 475b3074..461b488a 100644 --- a/docs/system_prompt.txt +++ b/docs/system_prompt.txt @@ -724,7 +724,7 @@ where $F_{obs}$ is the flux measured from the detector, $F_{transit}$ is the mod To model the transit lightcurve with {PyLightcurve}, considering the brightness variation from a star's edge (limb) to its center, EXOTIC generates nonlinear four-parameter limb darkening coefficients. The pipeline uses {ldtk} to calculate these coefficients from the star's temperature $T$, metallicity, surface gravity log $g$, and the observational filter based on PHOENIX stellar atmosphere models. ## Nested Sampler -{Ultranest} and {dynesty} are used for Bayesian inference and statistical analysis, each characterized by distinct implementation approaches and features. Nested sampling outperforms Markov Chain Monte Carlo (MCMC) in handling multi-modal and degenerate posteriors by not relying on a thermal transition property and avoiding the burn-in phase, making it a more efficient and robust data analysis method for astrophysical applications. Both nested sampling algorithms employ multiple ellipsoid bounds to outline the parameter space within which the free parameters can traverse. In {dynesty}, we opt to use the {DynamicNestedSampler} method due to its ability to adjust live points to distribute samples more efficiently, thereby speeding up computation. The choice of the nested sampler package depends on the availability of a C compiler and libraries. For users with Mac, Unix, Linux, or Windows systems with a C compiler installed, EXOTIC uses {ultranest}. For Windows systems lacking a C compiler, EXOTIC utilizes {dynesty}. +{Ultranest} is used for Bayesian inference and statistical analysis. Nested sampling outperforms Markov Chain Monte Carlo (MCMC) in handling multi-modal and degenerate posteriors by not relying on a thermal transition property and avoiding the burn-in phase, making it a more efficient and robust data analysis method for astrophysical applications. UltraNest employs multiple ellipsoid bounds to outline the parameter space within which the free parameters can traverse. EXOTIC uses {Ultranest} on Mac, Unix, Linux, and Windows systems. The nested sampling algorithm uses $T_{mid}$, $R_{p}/R_{s}$, and $i$ as free parameters with a uniform distribution in a bounded interval. Meanwhile, the sampler treats the remaining parameters ($a/R_s$, $e$, $w$, along with four-parameter limb darkening coefficients) as fixed. These free parameters are chosen based on their ability to constrain the ephemeris and estimate system parameters. Although we model the lightcurve using a least-squares fit with the Levenberg-Marquardt (LM) algorithm, we derive the final values and uncertainties using the nested sampler. If the Bayesian evidence stabilizes within a threshold (0.05) after each iteration, the algorithm has converged, indicating no further information can be gained from sampling. For well-constrained problems (i.e., each parameter having a single mode), convergence typically occurs within ~10,000-20,000 iterations. This efficiency makes nested sampling faster than Markov Chain Monte Carlo (MCMC), which usually requires around ~100,000 samples and lacks early stopping criteria. The corner plot shown in the triangle plot showcases histograms illustrating the marginalized posterior distributions for each parameter, accompanied by their estimated values and uncertainties. The sampler implements parameter constraints to ensure robust parameter exploration within established physical boundaries (e.g., the orbital inclination can not exceed $90^\circ$ as demonstrated in triangle plot. Scatter plots within the corner plot visually depict correlations among the free parameters, offering insights into their joint posterior distributions. Additionally, the corner plot facilitates the identification of parameter degeneracies, highlighting scenarios where alterations in one parameter impact another (see subplots in triangle plot that demonstrate degeneracies with their elliptical-shaped distributions). @@ -870,6 +870,7 @@ Example `inits.json` file: "Target Star DEC": "Must be in +/-DD:MM:SS sexagesimal format with correct sign at the beginning (+ or -).", "Demosaic Format": "Optional control for handling Bayer pattern color images - to use, provide Bayer color patttern of your camera (RGGB, BGGR, GRBG, GBRG) - null (no color processing) is default", "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", + "Boolean Values": "All boolean settings accept JSON true/false, numeric 1/0, or case-insensitive strings y/n. The equivalent strings yes/no and on/off are also accepted.", "Formatting of null": "Due to the file being a .json, null is case sensitive and must be spelled as shown.", "Decimal Format": "Leading zero must be included when appropriate (Ex: 0.32, .32 or 00.32 causes errors.)." }, @@ -892,11 +893,12 @@ Example `inits.json` file: "Filter Name (aavso.org/filters)": "CV", "Observing Notes": "Weather, seeing was nice.", - "Plate Solution? (y/n)": "y", - "Add Comparison Stars from AAVSO? (y/n)": "y", + "Plate Solution? (y/n)": true, + "Add Comparison Stars from AAVSO? (y/n)": false, "Target Star X & Y Pixel": "[424, 286]", "Comparison Star(s) X & Y Pixel": "[[465, 183], [512, 263], [], [], [], [], [], [], [], []]", + "Comparison Star(s) RA & Dec": null, "Demosaic Format": null, "Demosaic Output": null diff --git a/examples/single_transit/transit_fit_example.py b/examples/single_transit/transit_fit_example.py index 63c1caf8..97e9a548 100644 --- a/examples/single_transit/transit_fit_example.py +++ b/examples/single_transit/transit_fit_example.py @@ -12,8 +12,8 @@ 'ecc': 0.5, # Eccentricity 'omega': 120, # Arg of periastron 'tmid': 0.75, # Time of mid transit [day], - 'a1': 50, # Airmass coefficients - 'a2': 0., # trend = a1 * np.exp(a2 * airmass) + 'a0': 50, # Baseline flux normalization + 'a2': 0., # trend = a0 * np.exp(a2 * airmass) 'T*':5000, 'FE/H': 0, @@ -36,8 +36,8 @@ airmass = np.zeros(time.shape[0]) # GENERATE NOISY DATA - data = transit(time, prior)*prior['a1']*np.exp(prior['a2']*airmass) - data += np.random.normal(0, prior['a1']*250e-6, len(time)) + data = transit(time, prior)*prior['a0']*np.exp(prior['a2']*airmass) + data += np.random.normal(0, prior['a0']*250e-6, len(time)) dataerr = np.random.normal(300e-6, 50e-6, len(time)) + np.random.normal(300e-6, 50e-6, len(time)) # add optimization bounds for free parameters only @@ -45,14 +45,15 @@ 'rprs': [0, 0.1], 'tmid': [prior['tmid']-0.01, prior['tmid']+0.01], 'inc': [87,90], + #'a0': [0.95 * prior['a0'], 1.05 * prior['a0']], # optional explicit baseline offset #'a2': [0, 0.3] # uncomment if you want to fit for airmass - # a2 is used for individual airmass detrending using: a1*exp(airmass*a2) - # a1 is solved for automatically using mean(data/model) and does not need + # a2 is used for individual airmass detrending using: a0*exp(airmass*a2) + # a0 is optional. If omitted, the normalization is solved analytically. + # a1 is kept as a legacy alias for the resolved normalization. # to be included as a free parameter. A monte carlo process is used after # fitting to derive uncertainties on it. It acts like a normalization factor. - # never list 'a1' in bounds, it is perfectly correlated to exp(a2*airmass) - # and is solved for during the fit + # never list both 'a0' and 'a1' in bounds because they are the same scale term } # call the fitting routine @@ -98,4 +99,4 @@ test_ld(ld_obj, filter_info) ld = [ld_obj.ld0[0], ld_obj.ld1[0], ld_obj.ld2[0], ld_obj.ld3[0]] prior['u0'],prior['u1'],prior['u2'],prior['u3'] = ld - """ \ No newline at end of file + """ diff --git a/examples/tess/candidates/for_exotic_py_candidate_inits_maker.py b/examples/tess/candidates/for_exotic_py_candidate_inits_maker.py index fd29bcbb..e000c14b 100644 --- a/examples/tess/candidates/for_exotic_py_candidate_inits_maker.py +++ b/examples/tess/candidates/for_exotic_py_candidate_inits_maker.py @@ -538,6 +538,7 @@ def create_inits_file(parameters, file_name): "Directory of Biases": parameters.get("Directory of Biases", None), "AAVSO Observer Code (N/A if none)": parameters.get("AAVSO Observer Code (N/A if none)", "N/A"), "Secondary Observer Codes (N/A if none)": parameters.get("Secondary Observer Codes (N/A if none)", "N/A"), + "Observatory Full Title": parameters.get("Observatory Full Title", ""), "Observation date": parameters.get("Observation date", None), "Obs. Latitude": parameters.get("Obs. Latitude", None), "Obs. Longitude": parameters.get("Obs. Longitude", None), @@ -547,14 +548,28 @@ def create_inits_file(parameters, file_name): "Filter Name (aavso.org/filters)": parameters.get("Filter Name (aavso.org/filters)", None), "Observing Notes": parameters.get("Observing Notes", "N/A"), "Plate Solution? (y/n)": parameters.get("Plate Solution? (y/n)", None), - "Align Images? (y/n)": parameters.get("Align Images? (y/n)", None), "Target Star X & Y Pixel": parameters.get("Target Star X & Y Pixel", None), - "Comparison Star(s) X & Y Pixel": parameters.get("Comparison Star(s) X & Y Pixel", None) + "Comparison Star(s) X & Y Pixel": parameters.get("Comparison Star(s) X & Y Pixel", None), + "Comparison Star(s) RA & Dec": parameters.get("Comparison Star(s) RA & Dec", None) }, "optional_info": { "Pixel Scale (Ex: 5.21 arcsecs/pixel)": parameters.get("Pixel Scale (Ex: 5.21 arcsecs/pixel)", None), "Filter Minimum Wavelength (nm)": parameters.get("Filter Minimum Wavelength (nm)", None), - "Filter Maximum Wavelength (nm)": parameters.get("Filter Maximum Wavelength (nm)", None) + "Filter Maximum Wavelength (nm)": parameters.get("Filter Maximum Wavelength (nm)", None), + "disable vertical flux normalization": parameters.get("disable vertical flux normalization", False), + "maximum_number_of_ensemble_comparisons_for_transit": parameters.get( + "maximum_number_of_ensemble_comparisons_for_transit", 5 + ), + "maximum_number_of_ensemble_comparisons_for_stellar_variability": parameters.get( + "maximum_number_of_ensemble_comparisons_for_stellar_variability", 5 + ), + "require_apparent_magnitudes": parameters.get( + "require_apparent_magnitudes", True + ), + "use_exactly_the_comps_provided": parameters.get( + "use_exactly_the_comps_provided", False + ), + "require_comp_star": parameters.get("require_comp_star", "y") } } # Update the filename to include the planet name @@ -669,7 +684,6 @@ def create_inits_file(parameters, file_name): filter_name = input("Enter filter name: ").strip() observing_notes = input("Enter observing notes (or 'N/A' if none): ").strip() plate_solution = input("Plate solution? (y/n): ").strip() - align_images = input("Align images? (y/n): ").strip() target_star_xy = [int(coord) for coord in input("Enter target star X & Y Pixel (comma separated): ").strip().split(',')] comparison_stars_xy = [ [int(coord) for coord in star.strip().split(',')] @@ -695,7 +709,6 @@ def create_inits_file(parameters, file_name): stored_parameters["Filter Name (aavso.org/filters)"] = filter_name stored_parameters["Observing Notes"] = observing_notes stored_parameters["Plate Solution? (y/n)"] = plate_solution - stored_parameters["Align Images? (y/n)"] = align_images stored_parameters["Target Star X & Y Pixel"] = target_star_xy stored_parameters["Comparison Star(s) X & Y Pixel"] = comparison_stars_xy stored_parameters["Pixel Scale (Ex: 5.21 arcsecs/pixel)"] = pixel_scale diff --git a/examples/tess/candidates/toi.py b/examples/tess/candidates/toi.py index 09d6b766..58cb7df9 100644 --- a/examples/tess/candidates/toi.py +++ b/examples/tess/candidates/toi.py @@ -27,8 +27,14 @@ from wotan import flatten from exotic.api.elca import transit, lc_fitter from exotic.api.output_aavso import OutputFiles +from exotic.utils import safe_output_filename from transitleastsquares import transitleastsquares + +def output_file_path(output_dir, prefix, *parts, extension): + return os.path.join(output_dir, safe_output_filename(prefix, *parts, extension=extension)) + + def tap_query(base_url, query, dataframe=True): # Table Access Protocol (TAP) query uri_full = base_url @@ -222,7 +228,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # aperture plot tpf.plot(aperture_mask=aper_final) - plt.savefig(os.path.join(planetdir, planetname + f"_sector_{sector}_aperture.png")) + plt.savefig(output_file_path(planetdir, planetname, f"sector_{sector}", "aperture", extension="png")) plt.close() # remove first ~30 min of data after any big gaps @@ -285,7 +291,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] ax.set_xlabel("Time [TBJD]") ax.set_ylim([np.percentile(flux, 0.1), np.percentile(flux, 99.9)]) plt.tight_layout() - plt.savefig(os.path.join(planetdir, planetname + f"_sector_{sector}_trend.png")) + plt.savefig(output_file_path(planetdir, planetname, f"sector_{sector}", "trend", extension="png")) plt.close() # combine all sectors @@ -305,7 +311,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # create dataframe for entire light curve df = pd.DataFrame({'time': time, 'flux': flux * trend, 'flux_err': flux_err * trend, 'sector': alls}) - df.to_csv(os.path.join(planetdir, planetname + "_lightcurve.csv"), index=False) + df.to_csv(output_file_path(planetdir, planetname, "lightcurve", extension="csv"), index=False) # fit transit for each epoch period = prior['pl_orbper'] @@ -391,11 +397,11 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # create plots fig, ax = myfit.plot_bestfit(title=f"{prior['pl_name']} Global Fit") ax[0].set_ylim([np.percentile(flux, 1) * 0.99, np.percentile(flux, 99) * 1.01]) - plt.savefig(os.path.join(planetdir, planetname + "_global_fit.png")) + plt.savefig(output_file_path(planetdir, planetname, "global_fit", extension="png")) plt.close() myfit.plot_triangle() - plt.savefig(os.path.join(planetdir, planetname + "_global_triangle.png")) + plt.savefig(output_file_path(planetdir, planetname, "global_triangle", extension="png")) plt.close() # update priors from best fit @@ -412,7 +418,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] prior['pl_orbinclerr2'] = -myfit.errors['inc'] # save prior to disk - with open(os.path.join(planetdir, planetname + "_prior.json"), 'w', encoding='utf8') as json_file: + with open(output_file_path(planetdir, planetname, "prior", extension="json"), 'w', encoding='utf8') as json_file: json.dump(prior, json_file, indent=4) # save results to state vector @@ -444,7 +450,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] plt.xlabel('Period (days)') plt.plot(results.periods, results.power, color='black', lw=0.5) plt.xlim(0, max(results.periods)) - plt.savefig(os.path.join(planetdir, planetname + "_periodogram.png")) + plt.savefig(output_file_path(planetdir, planetname, "periodogram", extension="png")) plt.close() # save global fit data if above certain SNR @@ -455,7 +461,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] if snr < 1: with open("notes.txt", 'w') as f: f.write(f"Skipping individual light curve fits b.c SNR = {snr:.2f}") - pickle.dump(sv, open(os.path.join(planetdir, planetname + "_data.pkl"), "wb")) + pickle.dump(sv, open(output_file_path(planetdir, planetname, "data", extension="pkl"), "wb")) raise(Exception(f"Skipping individual light curve fits b.c SNR = {snr:.2f}")) period = myfit.parameters['per'] @@ -536,15 +542,15 @@ def check_std(time, flux, dt=0.5): # dt = [hr] tmidstr = str(np.round(myfit.parameters['tmid'], 2)).replace('.', '_') fig, ax = myfit.plot_bestfit(title=f"{prior['pl_name']} - Sector {lcdata['sector']}", bin_dt=0.5 / 24.) - plt.savefig(os.path.join(planetdir, f"{tmidstr}_" + planetname + "_lightcurve.png")) + plt.savefig(output_file_path(planetdir, tmidstr, planetname, "lightcurve", extension="png")) plt.close() fig = myfit.plot_triangle() - plt.savefig(os.path.join(planetdir, f"{tmidstr}_" + planetname + "_posterior.png")) + plt.savefig(output_file_path(planetdir, tmidstr, planetname, "posterior", extension="png")) plt.close() csv_lk = OutputFiles(myfit, prior, infoDict, planetdir) csv_lk.aavso_csv(airmass, u0, u1, u2, u3, tmidstr) csv_lk.aavso(airmass, u0, u1, u2, u3, tmidstr) - pickle.dump(sv, open(os.path.join(planetdir, planetname + "_data.pkl"), "wb")) + pickle.dump(sv, open(output_file_path(planetdir, planetname, "data", extension="pkl"), "wb")) diff --git a/examples/tess/candidates/toi_individ_lc.py b/examples/tess/candidates/toi_individ_lc.py index 237dfe92..615e7f7f 100644 --- a/examples/tess/candidates/toi_individ_lc.py +++ b/examples/tess/candidates/toi_individ_lc.py @@ -27,8 +27,14 @@ from wotan import flatten from exotic.api.elca import transit, lc_fitter from exotic.api.output_aavso import OutputFiles +from exotic.utils import safe_output_filename from transitleastsquares import transitleastsquares + +def output_file_path(output_dir, prefix, *parts, extension): + return os.path.join(output_dir, safe_output_filename(prefix, *parts, extension=extension)) + + def sigma_clip(ogdata, dt, iterations=1): mask = np.ones(ogdata.shape, dtype=bool) for i in range(iterations): @@ -203,7 +209,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # aperture plot tpf.plot(aperture_mask=aper_final) - plt.savefig(os.path.join(planetdir, planetname + f"_sector_{sector}_aperture.png")) + plt.savefig(output_file_path(planetdir, planetname, f"sector_{sector}", "aperture", extension="png")) plt.close() # remove first ~30 min of data after any big gaps @@ -267,7 +273,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] ax.set_xlabel("Time [TBJD]") ax.set_ylim([np.percentile(flux, 0.1), np.percentile(flux, 99.9)]) plt.tight_layout() - plt.savefig(os.path.join(planetdir, planetname + f"_sector_{sector}_trend.png")) + plt.savefig(output_file_path(planetdir, planetname, f"sector_{sector}", "trend", extension="png")) plt.close() # combine all sectors @@ -287,7 +293,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # create dataframe for entire light curve df = pd.DataFrame({'time': time, 'flux': flux * trend, 'flux_err': flux_err * trend, 'sector': alls}) - df.to_csv(os.path.join(planetdir, planetname + "_lightcurve.csv"), index=False) + df.to_csv(output_file_path(planetdir, planetname, "lightcurve", extension="csv"), index=False) # fit transit for each epoch period = prior['pl_orbper'] @@ -458,11 +464,11 @@ def check_std(time, flux, dt=0.5): # dt = [hr] tmidstr = str(np.round(myfit.parameters['tmid'], 2)).replace('.', '_') fig, ax = myfit.plot_bestfit(title=f"{prior['pl_name']} - Sector {lcdata['sector']}", bin_dt=0.5 / 24.) - plt.savefig(os.path.join(planetdir, f"{tmidstr}_" + planetname + "_lightcurve.png")) + plt.savefig(output_file_path(planetdir, tmidstr, planetname, "lightcurve", extension="png")) plt.close() fig = myfit.plot_triangle() - plt.savefig(os.path.join(planetdir, f"{tmidstr}_" + planetname + "_posterior.png")) + plt.savefig(output_file_path(planetdir, tmidstr, planetname, "posterior", extension="png")) plt.close() csv_data = { @@ -475,4 +481,4 @@ def check_std(time, flux, dt=0.5): # dt = [hr] csv_lk.aavso_csv(airmass, u0, u1, u2, u3, tmidstr) csv_lk.aavso(airmass, u0, u1, u2, u3, tmidstr) - pickle.dump(sv, open(os.path.join(planetdir, planetname + "_data.pkl"), "wb")) + pickle.dump(sv, open(output_file_path(planetdir, planetname, "data", extension="pkl"), "wb")) diff --git a/examples/tess/tess.py b/examples/tess/tess.py index db43e8c3..f8017589 100644 --- a/examples/tess/tess.py +++ b/examples/tess/tess.py @@ -33,8 +33,14 @@ from wotan import flatten from exotic.api.elca import transit, lc_fitter from exotic.api.output_aavso import OutputFiles +from exotic.utils import safe_output_filename from transitleastsquares import transitleastsquares + +def output_file_path(output_dir, prefix, *parts, extension): + return os.path.join(output_dir, safe_output_filename(prefix, *parts, extension=extension)) + + def tap_query(base_url, query, dataframe=True): # table access protocol query @@ -166,8 +172,9 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # https://exo.mast.stsci.edu/ # load prior from disk or download - if os.path.exists(os.path.join(planetdir,planetname+"_prior.json")): - prior = json.load(open(os.path.join(planetdir,planetname+"_prior.json"),"r")) + prior_path = output_file_path(planetdir, planetname, "prior", extension="json") + if os.path.exists(prior_path): + prior = json.load(open(prior_path, "r")) else: # download prior from web if "TOI" in args.target: @@ -302,7 +309,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] #os.mkdir(os.path.join(planetdir,"lightcurves")) # save prior to disk - with open(os.path.join(planetdir,planetname+"_prior.json"), 'w', encoding ='utf8') as json_file: + with open(prior_path, 'w', encoding ='utf8') as json_file: json.dump(prior, json_file, indent=4) if len(prior) == 0: @@ -395,7 +402,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # aperture plot tpf.plot(aperture_mask=aper_final) - plt.savefig( os.path.join(planetdir, planetname+f"_sector_{sector}_aperture.png") ) + plt.savefig(output_file_path(planetdir, planetname, f"sector_{sector}", "aperture", extension="png")) plt.close() # remove first ~30 min of data after any big gaps @@ -467,7 +474,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] plt.tight_layout() #if not os.path.exists(os.path.join(planetdir, "lightcurves")): #os.makedirs(os.path.join(planetdir, "lightcurves")) - plt.savefig( os.path.join(planetdir, planetname+f"_sector_{sector}_trend.png") ) + plt.savefig(output_file_path(planetdir, planetname, f"sector_{sector}", "trend", extension="png")) plt.close() # combine all sectors @@ -487,7 +494,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # create dataframe for entire light curve df = pd.DataFrame({'time':time, 'flux':flux*trend, 'flux_err':flux_err*trend, 'sector':alls}) - df.to_csv( os.path.join(planetdir, planetname+"_lightcurve.csv"), index=False) + df.to_csv(output_file_path(planetdir, planetname, "lightcurve", extension="csv"), index=False) # fit transit for each epoch period = prior['pl_orbper'] @@ -576,11 +583,11 @@ def check_std(time, flux, dt=0.5): # dt = [hr] fig,ax = myfit.plot_bestfit(title=f"{args.target} Global Fit") # set y_limit between 1 and 99 percentile ax[0].set_ylim([np.percentile(flux, 1)*0.99, np.percentile(flux,99)*1.01]) - plt.savefig( os.path.join( planetdir, planetname+"_global_fit.png")) + plt.savefig(output_file_path(planetdir, planetname, "global_fit", extension="png")) plt.close() myfit.plot_triangle() - plt.savefig( os.path.join( planetdir, planetname+"_global_triangle.png")) + plt.savefig(output_file_path(planetdir, planetname, "global_triangle", extension="png")) plt.close() # update priors @@ -606,7 +613,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] prior['pl_orbinclerr2'] = -myfit.errors['inc'] # save prior to disk - with open(os.path.join(planetdir,planetname+"_prior.json"), 'w', encoding ='utf8') as json_file: + with open(prior_path, 'w', encoding ='utf8') as json_file: json.dump(prior, json_file, indent=4) # save results to state vector @@ -639,7 +646,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] plt.xlabel('Period (days)') plt.plot(results.periods, results.power, color='black', lw=0.5) plt.xlim(0, max(results.periods)) - plt.savefig( os.path.join( planetdir, planetname+"_periodogram.png")) + plt.savefig(output_file_path(planetdir, planetname, "periodogram", extension="png")) plt.close() sv['tls'] = { @@ -660,7 +667,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] with open("notes.txt", 'w') as f: f.write(f"Skipping individual light curve fits b.c SNR = {snr:.2f}") # save global fit data - pickle.dump(sv, open(os.path.join(planetdir, planetname+"_data.pkl"),"wb")) + pickle.dump(sv, open(output_file_path(planetdir, planetname, "data", extension="pkl"), "wb")) raise(Exception(f"Skipping individual light curve fits b.c SNR = {snr:.2f}")) # prepare for individual fits @@ -761,12 +768,12 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # save bestfit fig,ax = myfit.plot_bestfit(title=f"{args.target} - Sector {lcdata['sector']}", bin_dt=0.5/24.) - plt.savefig( os.path.join(planetdir, f"{tmidstr}_"+planetname+"_lightcurve.png") ) + plt.savefig(output_file_path(planetdir, tmidstr, planetname, "lightcurve", extension="png")) plt.close() # save posterior fig = myfit.plot_triangle() - plt.savefig( os.path.join(planetdir, f"{tmidstr}_"+planetname+"_posterior.png") ) + plt.savefig(output_file_path(planetdir, tmidstr, planetname, "posterior", extension="png")) plt.close() csv_data = { @@ -781,4 +788,4 @@ def check_std(time, flux, dt=0.5): # dt = [hr] csv_lk.aavso(airmass,u0,u1,u2,u3, tmidstr) # save sv pickle - pickle.dump(sv, open(os.path.join(planetdir, planetname+"_data.pkl"),"wb")) \ No newline at end of file + pickle.dump(sv, open(output_file_path(planetdir, planetname, "data", extension="pkl"), "wb")) diff --git a/examples/tess/tess_individ_lc.py b/examples/tess/tess_individ_lc.py index b06e33d2..a96b720d 100644 --- a/examples/tess/tess_individ_lc.py +++ b/examples/tess/tess_individ_lc.py @@ -27,8 +27,14 @@ from wotan import flatten from exotic.api.elca import transit, lc_fitter from exotic.api.output_aavso import OutputFiles +from exotic.utils import safe_output_filename from transitleastsquares import transitleastsquares + +def output_file_path(output_dir, prefix, *parts, extension): + return os.path.join(output_dir, safe_output_filename(prefix, *parts, extension=extension)) + + def tap_query(base_url, query, dataframe=True): # table access protocol query @@ -151,8 +157,9 @@ def check_std(time, flux, dt=0.5): # dt = [hr] raise(Exception(f"no data for: {args.target}")) # https://exo.mast.stsci.edu/ # load prior from disk or download - if os.path.exists(os.path.join(planetdir,planetname+"_prior.json")): - prior = json.load(open(os.path.join(planetdir,planetname+"_prior.json"),"r")) + prior_path = output_file_path(planetdir, planetname, "prior", extension="json") + if os.path.exists(prior_path): + prior = json.load(open(prior_path, "r")) else: # download prior from web if "TOI" in args.target: @@ -275,7 +282,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] #os.mkdir(os.path.join(planetdir,"lightcurves")) # save prior to disk - with open(os.path.join(planetdir,planetname+"_prior.json"), 'w', encoding ='utf8') as json_file: + with open(prior_path, 'w', encoding ='utf8') as json_file: json.dump(prior, json_file, indent=4) if len(prior) == 0: @@ -368,7 +375,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # aperture plot tpf.plot(aperture_mask=aper_final) - plt.savefig( os.path.join(planetdir, planetname+f"_sector_{sector}_aperture.png") ) + plt.savefig(output_file_path(planetdir, planetname, f"sector_{sector}", "aperture", extension="png")) plt.close() # remove first ~30 min of data after any big gaps @@ -435,7 +442,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] plt.tight_layout() #if not os.path.exists(os.path.join(planetdir, "lightcurves")): #os.makedirs(os.path.join(planetdir, "lightcurves")) - plt.savefig( os.path.join(planetdir, planetname+f"_sector_{sector}_trend.png") ) + plt.savefig(output_file_path(planetdir, planetname, f"sector_{sector}", "trend", extension="png")) plt.close() # combine all sectors @@ -455,7 +462,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # create dataframe for entire light curve df = pd.DataFrame({'time':time, 'flux':flux*trend, 'flux_err':flux_err*trend, 'sector':alls}) - df.to_csv( os.path.join(planetdir, planetname+"_lightcurve.csv"), index=False) + df.to_csv(output_file_path(planetdir, planetname, "lightcurve", extension="csv"), index=False) # fit transit for each epoch period = prior['pl_orbper'] @@ -635,12 +642,12 @@ def check_std(time, flux, dt=0.5): # dt = [hr] # save bestfit fig,ax = myfit.plot_bestfit(title=f"{args.target} - Sector {lcdata['sector']}", bin_dt=0.5/24.) - plt.savefig( os.path.join(planetdir, f"{tmidstr}_"+planetname+"_lightcurve.png") ) + plt.savefig(output_file_path(planetdir, tmidstr, planetname, "lightcurve", extension="png")) plt.close() # save posterior fig = myfit.plot_triangle() - plt.savefig( os.path.join(planetdir, f"{tmidstr}_"+planetname+"_posterior.png") ) + plt.savefig(output_file_path(planetdir, tmidstr, planetname, "posterior", extension="png")) plt.close() csv_data = { @@ -658,7 +665,7 @@ def check_std(time, flux, dt=0.5): # dt = [hr] print(f"Failed to create AAVSO csv for {args.target} - Sector {lcdata['sector']}") # save sv pickle - pickle.dump(sv, open(os.path.join(planetdir, planetname+"_data.pkl"),"wb")) + pickle.dump(sv, open(output_file_path(planetdir, planetname, "data", extension="pkl"), "wb")) # O-C plot tmids = np.array([lc['pars']['tmid'] for lc in sv['lightcurves']]) @@ -674,4 +681,4 @@ def check_std(time, flux, dt=0.5): # dt = [hr] ratios = ratios[dmask] # TODO finish making O-C plot? - # use example from exotic.api.nested_linear_fitter \ No newline at end of file + # use example from exotic.api.nested_linear_fitter diff --git a/exotic/__init__.py b/exotic/__init__.py index 1bf9216e..1861dc23 100644 --- a/exotic/__init__.py +++ b/exotic/__init__.py @@ -44,6 +44,7 @@ import importlib_metadata as metadata # Python <3.8 from pathlib import Path +from importlib import import_module import sys # Extend PYTHONPATH to include current directory and parent directory @@ -71,4 +72,41 @@ __version__ = version_read("exotic.py") except IOError: # Unable to read from exotic script - __version__ = "unknown" \ No newline at end of file + __version__ = "unknown" + + +def _load_runtime_callable(name): + """Load CLI entry points lazily without importing the full runtime at package import.""" + current_module = sys.modules.get(__name__) + module_names = [f"{__name__}.exotic", "exotic.exotic"] + + seen = set() + for module_name in module_names: + if module_name in seen: + continue + seen.add(module_name) + + try: + module = import_module(module_name) + except ModuleNotFoundError as exc: + if exc.name == module_name: + continue + raise + if module is current_module: + continue + runtime_callable = getattr(module, name, None) + if runtime_callable is not None: + return runtime_callable + + raise ImportError(f"cannot import name '{name}' from '{__name__}'") + + +def main(*args, **kwargs): + return _load_runtime_callable("main")(*args, **kwargs) + + +def cli(*args, **kwargs): + return _load_runtime_callable("cli")(*args, **kwargs) + + +__all__ = ["__version__", "main", "cli"] diff --git a/exotic/api/colab.py b/exotic/api/colab.py index c7662d0f..1e35d67f 100644 --- a/exotic/api/colab.py +++ b/exotic/api/colab.py @@ -53,27 +53,21 @@ ######################################################### from astropy.io import fits from astropy.time import Time -from barycorrpy import utc_tdb # import bokeh.io # from bokeh.io import output_notebook -from bokeh.palettes import Viridis256 -from bokeh.plotting import figure, output_file, show -from bokeh.models import BoxZoomTool, ColorBar, FreehandDrawTool, HoverTool, LinearColorMapper, LogColorMapper, \ +from bokeh.plotting import figure, show +from bokeh.models import BoxZoomTool, ColorBar, FreehandDrawTool, HoverTool, LogColorMapper, \ LogTicker, PanTool, ResetTool, WheelZoomTool # import copy -from io import BytesIO from IPython.display import display, HTML # from IPython.display import Image # from ipywidgets import widgets, HBox import json import numpy as np import os -from pprint import pprint import re -from scipy.ndimage import label -from skimage.transform import rescale, resize, downscale_local_mean +from skimage.transform import downscale_local_mean # import subprocess -import time def display_image(filename): @@ -188,20 +182,42 @@ def get_val(hdr, ks): ######################################################### def process_lat_long(val, key): - m = re.search(r"\'?([+-]?\d+)[\s\:](\d+)[\s\:](\d+\.?\d*)", val) - if m: - deg, min, sec = float(m.group(1)), float(m.group(2)), float(m.group(3)) - if deg < 0: - v = deg - (((60*min) + sec)/3600) - else: - v = deg + (((60*min) + sec)/3600) - return(add_sign(v)) - m = re.search("^\'?([+-]?\d+\.\d+)", val) - if m: - v = float(m.group(1)) - return(add_sign(v)) - else: + text = str(val).strip() + coordinate_type = str(key).strip().lower() + valid_hemispheres = { + "latitude": {"N", "S"}, + "longitude": {"E", "W"}, + }.get(coordinate_type) + hemisphere = None + trailing_hemisphere = re.search(r"([NSEW])\s*$", text) + leading_hemisphere = re.match(r"\s*([NSEW])(?=\s|[+-]?\d)", text) + hemisphere_match = trailing_hemisphere or leading_hemisphere + if hemisphere_match: + hemisphere = hemisphere_match.group(1) + if valid_hemispheres is not None and hemisphere not in valid_hemispheres: + print(f"Cannot match value {val}, which is meant to be {key}.") + return None + start, end = hemisphere_match.span(1) + text = f"{text[:start]}{text[end:]}".strip() + + number_tokens = re.findall(r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)", text) + if not 1 <= len(number_tokens) <= 3: + print(f"Cannot match value {val}, which is meant to be {key}.") + return None + if (len(number_tokens) == 1 and hemisphere is None and + "." not in number_tokens[0] and number_tokens[0][0] not in "+-"): print(f"Cannot match value {val}, which is meant to be {key}.") + return None + + degrees = float(number_tokens[0]) + minutes = abs(float(number_tokens[1])) if len(number_tokens) >= 2 else 0.0 + seconds = abs(float(number_tokens[2])) if len(number_tokens) >= 3 else 0.0 + magnitude = abs(degrees) + minutes / 60.0 + seconds / 3600.0 + if hemisphere: + sign = -1.0 if hemisphere in {"S", "W"} else 1.0 + else: + sign = -1.0 if number_tokens[0].startswith("-") else 1.0 + return(add_sign(sign * magnitude)) ######################################################### @@ -359,6 +375,7 @@ def make_inits_file(planetary_params, image_dir, output_dir, first_image, targ_c "AAVSO Observer Code (N/A if none)": "%s", "Secondary Observer Codes (N/A if none)": "%s", + "Observatory Full Title": "", "Observation date": "%s", "Obs. Latitude": "%s", @@ -370,10 +387,11 @@ def make_inits_file(planetary_params, image_dir, output_dir, first_image, targ_c "Observing Notes": "%s", "Plate Solution? (y/n)": "y", - "Add Comparison Stars from AAVSO? (y/n)": "y", + "Add Comparison Stars from AAVSO? (y/n)": "n", "Target Star X & Y Pixel": %s, "Comparison Star(s) X & Y Pixel": %s, + "Comparison Star(s) RA & Dec": null, "Demosaic Format": null, "Demosaic Output": null @@ -381,7 +399,26 @@ def make_inits_file(planetary_params, image_dir, output_dir, first_image, targ_c "optional_info": { "Pixel Scale (Ex: 5.21 arcsecs/pixel)": null, "Filter Minimum Wavelength (nm)": %s, - "Filter Maximum Wavelength (nm)": %s + "Filter Maximum Wavelength (nm)": %s, + "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, + "allow_pixel_alignment_fallback": true, + "bad_wcs_threshold_percent": 3.0, + "detrend_on_outoftransit_baseline": true, + "use_eebls_to_initialize_tmid_and_bounds": "y", + "pick_comparison_by_eebls_snr": "y", + "use_impactparameter_rather_than_inclination_to_fit": "y", + "maximum_number_of_ensemble_comparisons_for_transit": 5, + "maximum_number_of_ensemble_comparisons_for_stellar_variability": 5, + "require_apparent_magnitudes": true, + "use_exactly_the_comps_provided": false, + "use_adaptive_apertures": false, + "gain_electrons_per_adu": null, + "read_noise_electrons": null, + "dark_current_electrons_per_second_per_pixel": null, + "flat_field_fractional_error": null, + "telescope_aperture_m": null, + "scintillation_coefficient": null, + "require_comp_star": "y" } } """ % (planetary_params, image_dir, output_dir, flats_dir, darks_dir, biases_dir, diff --git a/exotic/api/elca.py b/exotic/api/elca.py index 093f9cd7..9e75443b 100644 --- a/exotic/api/elca.py +++ b/exotic/api/elca.py @@ -41,26 +41,99 @@ # ########################################################################### # from astropy.time import Time import copy -from itertools import cycle +from contextlib import redirect_stderr, redirect_stdout +import faulthandler +import io +from itertools import cycle, product +import math +import os +import sys +import bottleneck as bn import matplotlib.pyplot as plt import numpy as np -from pylightcurve.models.exoplanet_lc import transit as pytransit from scipy import spatial from scipy.optimize import least_squares from scipy.signal import savgol_filter +from scipy.special import ndtr, ndtri +from ultranest import ReactiveNestedSampler + try: - from ultranest import ReactiveNestedSampler + from ..utils import format_value_and_uncertainty, format_value_with_uncertainty except ImportError: - import dynesty - import dynesty.plotting - from dynesty.utils import resample_equal - from scipy.stats import gaussian_kde + from utils import format_value_and_uncertainty, format_value_with_uncertainty try: from plotting import corner except ImportError: from .plotting import corner +try: + from ultranest_utils import run_reactive_sampler +except ImportError: + from .ultranest_utils import run_reactive_sampler + +BAD_LOG_LIKELIHOOD = -1.0e100 +TRIANGLE_PLOT_EDGE_PEAK_FRACTION_MAX = 0.50 +TRIANGLE_PLOT_EDGE_MIN_SAMPLE_COUNT = 30 +TRIANGLE_PLOT_EDGE_EXPANSION_STEPS = 8 +TRIANGLE_PLOT_FALLBACK_EXPANSION_BOUNDS = { + 'rprs': (0.0, 1.0), +} +TRANSIT_MODEL_UNCERTAINTY_KEYS = ( + 'rprs', 'tmid', 'inc', 'ars', 'per', 'ecc', 'omega', 'u0', 'u1', 'u2', 'u3', +) +BASELINE_MODEL_UNCERTAINTY_KEYS = ('a0', 'a1', 'a2') +MODEL_UNCERTAINTY_POSTERIOR_SAMPLE_LIMIT = 2000 +EXPOSURE_SMEARING_EXPOSURE_TIME_KEY = '_exposure_time_days' +EXPOSURE_SMEARING_SUPERSAMPLE_KEY = '_exposure_smearing_supersample' +EXPOSURE_SMEARING_CHANGE_TOLERANCE_KEY = '_exposure_smearing_change_tolerance' +DEFAULT_EXPOSURE_SMEARING_SUPERSAMPLE = 7 +DEFAULT_EXPOSURE_SMEARING_CHANGE_TOLERANCE = 1.0e-5 +MIN_EXPOSURE_SMEARING_SECONDS = 1.0 +EXPOSURE_SMEARING_TRANSIT_WINDOW_PADDING_FACTOR = 2.0 +ULTRANEST_INFLATED_ERROR_REPLACEMENT_FACTOR = 3.0 +ULTRANEST_LOCAL_UNCERTAINTY_MAX_DELTA_CHI2 = 9.0 +ULTRANEST_EXPANDED_PRIOR_WARMSTART_FULL_PRIOR_FRACTION = 0.5 +ULTRANEST_EXPANDED_PRIOR_WARMSTART_MINIMUM_SAMPLE_COUNT = 32 +ULTRANEST_EXPANDED_PRIOR_WARMSTART_MAXIMUM_SAMPLE_COUNT = 20000 + +def _pylightcurve_import_watchdog_seconds(): + try: + return float(os.environ.get("EXOTIC_IMPORT_WATCHDOG_SECONDS", "120")) + except (TypeError, ValueError): + return 120.0 + + +def _start_import_watchdog(): + timeout = _pylightcurve_import_watchdog_seconds() + if timeout <= 0: + return False + + try: + if not faulthandler.is_enabled(): + faulthandler.enable(file=sys.__stdout__, all_threads=True) + faulthandler.dump_traceback_later(timeout, repeat=True, file=sys.__stdout__) + return True + except Exception: + return False + + +def _load_pylightcurve_transit(): + watchdog_started = _start_import_watchdog() + try: + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + from pylightcurve.models.exoplanet_lc import transit + return transit + finally: + if watchdog_started: + try: + faulthandler.cancel_dump_traceback_later() + except Exception: + pass + + +pytransit = _load_pylightcurve_transit() + def weightedflux(flux, gw, nearest): return np.sum(flux[nearest] * gw, axis=-1) @@ -83,7 +156,7 @@ def gaussian_weights(X, w=1, neighbors=50, feature_scale=1000): return gw, nearest.astype(int) -def transit(times, values): +def _instantaneous_transit(times, values): model = pytransit([values['u0'], values['u1'], values['u2'], values['u3']], values['rprs'], values['per'], values['ars'], values['ecc'], values['inc'], values['omega'], @@ -91,15 +164,428 @@ def transit(times, values): return model +def _exposure_time_days_for_times(values, shape): + try: + exposure_time_days = values.get(EXPOSURE_SMEARING_EXPOSURE_TIME_KEY) + except AttributeError: + return None + if exposure_time_days is None: + return None + + try: + exposure_time_days = np.asarray(exposure_time_days, dtype=float) + except (TypeError, ValueError): + return None + + if exposure_time_days.shape == (): + exposure_time_days = np.full(shape, float(exposure_time_days), dtype=float) + elif exposure_time_days.shape != shape: + return None + else: + exposure_time_days = np.array(exposure_time_days, dtype=float, copy=True) + + minimum_exposure_days = MIN_EXPOSURE_SMEARING_SECONDS / 86400.0 + finite_positive = np.isfinite(exposure_time_days) & (exposure_time_days > minimum_exposure_days) + if not np.any(finite_positive): + return None + + exposure_time_days[~finite_positive] = 0.0 + return exposure_time_days + + +def _exposure_smearing_supersample(values): + try: + sample_count = int(values.get(EXPOSURE_SMEARING_SUPERSAMPLE_KEY, DEFAULT_EXPOSURE_SMEARING_SUPERSAMPLE)) + except (AttributeError, TypeError, ValueError): + sample_count = DEFAULT_EXPOSURE_SMEARING_SUPERSAMPLE + sample_count = max(3, sample_count) + if sample_count % 2 == 0: + sample_count += 1 + return sample_count + + +def _exposure_smearing_change_tolerance(values): + try: + tolerance = float(values.get( + EXPOSURE_SMEARING_CHANGE_TOLERANCE_KEY, + DEFAULT_EXPOSURE_SMEARING_CHANGE_TOLERANCE, + )) + except (AttributeError, TypeError, ValueError): + tolerance = DEFAULT_EXPOSURE_SMEARING_CHANGE_TOLERANCE + if not np.isfinite(tolerance) or tolerance < 0: + tolerance = DEFAULT_EXPOSURE_SMEARING_CHANGE_TOLERANCE + return tolerance + + +def _exposure_smearing_candidate_mask(times, exposure_time_days, values): + candidate = ( + np.isfinite(times) + & np.isfinite(exposure_time_days) + & (exposure_time_days > MIN_EXPOSURE_SMEARING_SECONDS / 86400.0) + ) + if not np.any(candidate): + return candidate + + try: + period = float(values['per']) + tmid = float(values['tmid']) + except (KeyError, TypeError, ValueError): + return candidate + if not np.isfinite(period) or period <= 0 or not np.isfinite(tmid): + return candidate + + duration = transit_duration(values) + if not np.isfinite(duration) or duration <= 0: + return candidate + + max_exposure = np.nanmax(exposure_time_days[candidate]) + if not np.isfinite(max_exposure) or max_exposure <= 0: + return np.zeros(times.shape, dtype=bool) + + half_window = ( + 0.5 * duration + + (0.5 + EXPOSURE_SMEARING_TRANSIT_WINDOW_PADDING_FACTOR) * max_exposure + ) + phase_days = get_phase(times[candidate], period, tmid) * period + narrowed = np.abs(phase_days) <= half_window + candidate_indices = np.flatnonzero(candidate) + candidate[candidate_indices] = narrowed + return candidate + + +def transit(times, values): + model = _instantaneous_transit(times, values) + exposure_time_days = _exposure_time_days_for_times(values, np.asarray(times).shape) + if exposure_time_days is None: + return model + + try: + times_array = np.asarray(times, dtype=float) + model_array = np.asarray(model, dtype=float) + except (TypeError, ValueError): + return model + if model_array.shape != times_array.shape: + return model + + flat_times = times_array.reshape(-1) + flat_model = model_array.reshape(-1).copy() + flat_exposure_time_days = exposure_time_days.reshape(-1) + candidate_mask = _exposure_smearing_candidate_mask( + flat_times, + flat_exposure_time_days, + values, + ) + candidate_indices = np.flatnonzero(candidate_mask) + if candidate_indices.size == 0: + return model_array + + candidate_times = flat_times[candidate_indices] + candidate_exposures = flat_exposure_time_days[candidate_indices] + half_exposures = 0.5 * candidate_exposures + try: + start_model = np.asarray( + _instantaneous_transit(candidate_times - half_exposures, values), + dtype=float, + ).reshape(-1) + end_model = np.asarray( + _instantaneous_transit(candidate_times + half_exposures, values), + dtype=float, + ).reshape(-1) + except Exception: + return model_array + if start_model.shape != candidate_times.shape or end_model.shape != candidate_times.shape: + return model_array + + center_model = flat_model[candidate_indices] + model_change = np.maximum.reduce(( + np.abs(start_model - center_model), + np.abs(end_model - center_model), + np.abs(end_model - start_model), + )) + active_indices = candidate_indices[model_change > _exposure_smearing_change_tolerance(values)] + if active_indices.size == 0: + return model_array + + sample_count = _exposure_smearing_supersample(values) + offsets = (np.arange(sample_count, dtype=float) + 0.5) / sample_count - 0.5 + active_times = flat_times[active_indices] + active_exposures = flat_exposure_time_days[active_indices] + sample_times = (active_times[:, None] + active_exposures[:, None] * offsets[None, :]).reshape(-1) + try: + sample_model = np.asarray(_instantaneous_transit(sample_times, values), dtype=float) + except Exception: + return model_array + if sample_model.size != active_indices.size * sample_count: + return model_array + + flat_model[active_indices] = sample_model.reshape(active_indices.size, sample_count).mean(axis=1) + return flat_model.reshape(model_array.shape) + + +def impact_parameter_scale(values): + ecc = values.get('ecc', 0.0) + omega = np.deg2rad(values.get('omega', 0.0)) + denom = 1.0 + ecc * np.sin(omega) + if np.any(np.isclose(denom, 0.0)): + denom = np.where(np.isclose(denom, 0.0), np.finfo(float).eps, denom) + return values['ars'] * (1.0 - ecc ** 2) / denom + + +def impact_parameter_from_inclination(values, inclination): + return impact_parameter_scale(values) * np.cos(np.deg2rad(inclination)) + + +def inclination_from_impact_parameter(values, impact_parameter): + scale = impact_parameter_scale(values) + if np.any(np.isclose(scale, 0.0)): + scale = np.where(np.isclose(scale, 0.0), np.finfo(float).eps, scale) + cosi = np.clip(np.asarray(impact_parameter, dtype=float) / scale, -1.0, 1.0) + return np.rad2deg(np.arccos(cosi)) + + +def grazing_impact_parameter(values): + try: + rprs = float(values['rprs']) + except (KeyError, TypeError, ValueError): + return np.nan + + if not np.isfinite(rprs) or rprs < 0: + return np.nan + return 1.0 + rprs + + +def transit_duration(values): + try: + period = float(values['per']) + rprs = float(values['rprs']) + ars = float(values['ars']) + inc = float(values['inc']) + except (KeyError, TypeError, ValueError): + return np.nan + + if ( + not np.isfinite(period) or period <= 0 + or not np.isfinite(rprs) or rprs < 0 + or not np.isfinite(ars) or ars <= 0 + or not np.isfinite(inc) + ): + return np.nan + + ecc = values.get('ecc', 0.0) + omega = np.deg2rad(values.get('omega', 0.0)) + sin_inc = np.sin(np.deg2rad(inc)) + if not np.isfinite(sin_inc) or sin_inc <= 0: + return np.nan + + impact_scale = ars * (1.0 - ecc ** 2) / max(np.finfo(float).eps, 1.0 + ecc * np.sin(omega)) + impact_parameter = impact_scale * np.cos(np.deg2rad(inc)) + chord_sq = (1.0 + rprs) ** 2 - impact_parameter ** 2 + if not np.isfinite(chord_sq) or chord_sq <= 0 or not np.isfinite(impact_scale) or impact_scale <= 0: + return np.nan + + argument = np.sqrt(chord_sq) / (impact_scale * sin_inc) + argument = float(np.clip(argument, -1.0, 1.0)) + duration = (period / np.pi) * np.arcsin(argument) + return float(duration) if np.isfinite(duration) and duration > 0 else np.nan + + def get_phase(times, per, tmid): return (times - tmid + 0.25 * per) / per % 1 - 0.25 -def mc_a1(m_a2, sig_a2, transit, airmass, data, n=10000): +def normalize_time_range(time_range): + if time_range is None: + return None + + values = np.asarray(time_range, dtype=float).reshape(-1) + finite = values[np.isfinite(values)] + if finite.size == 0: + return None + + return float(np.min(finite)), float(np.max(finite)) + + +def get_plot_phase(times, per, tmid, reference_times=None): + times = np.asarray(times, dtype=float) + if not np.isfinite(per) or per == 0: + return times * np.nan + + raw_phase = (times - tmid) / per + + reference_range = normalize_time_range(reference_times) + if reference_range is None: + finite_phase = raw_phase[np.isfinite(raw_phase)] + if finite_phase.size == 0: + return raw_phase + reference_epoch = float(np.rint(0.5 * (np.min(finite_phase) + np.max(finite_phase)))) + else: + ref_phase = (np.asarray(reference_range, dtype=float) - tmid) / per + reference_epoch = float(np.rint(np.mean(ref_phase))) + + return raw_phase - reference_epoch + + +def fallback_flux_baseline(): + return 1.0 + + +def has_explicit_flux_baseline(bounds): + return any(key in bounds for key in ('a0', 'a1')) + + +def get_flux_baseline(values, fallback=1.0): + if 'a0' in values: + return values['a0'] + if 'a1' in values: + return values['a1'] + return fallback + + +def get_airmass_reference(airmass): + airmass = np.asarray(airmass, dtype=float) + finite_airmass = airmass[np.isfinite(airmass)] + if finite_airmass.size == 0: + return 0.0 + return float(np.nanmean(finite_airmass)) + + +def center_airmass(airmass, reference=None): + airmass = np.asarray(airmass, dtype=float) + if reference is None: + reference = get_airmass_reference(airmass) + return airmass - float(reference) + + +def airmass_trend(a2, airmass, reference=None): + return np.exp(np.asarray(a2, dtype=float) * center_airmass(airmass, reference=reference)) + + +def airmass_trend_grid(a2_values, airmass, reference=None): + centered = center_airmass(airmass, reference=reference) + return np.exp(np.outer(np.asarray(a2_values, dtype=float), centered)) + + +def normalized_optional_fit_mask(mask, shape): + if mask is None: + return None + fit_mask = np.asarray(mask, dtype=bool) + if fit_mask.shape != tuple(shape): + return None + if not np.any(fit_mask): + return None + return fit_mask + + +def solve_flux_baseline(model, data, dataerr=None, mask=None): + model = np.asarray(model, dtype=float) + data = np.asarray(data, dtype=float) + weights = np.ones(model.shape, dtype=float) + fit_mask = normalized_optional_fit_mask(mask, model.shape) + + if dataerr is not None: + dataerr = np.asarray(dataerr, dtype=float) + weights = np.zeros(model.shape, dtype=float) + valid_err = np.isfinite(dataerr) & (dataerr > 0) + weights[valid_err] = 1.0 / (dataerr[valid_err] ** 2) + + mask = np.isfinite(model) & np.isfinite(data) & (model != 0) + if fit_mask is not None: + mask &= fit_mask + if dataerr is not None: + mask &= np.isfinite(weights) & (weights > 0) + + if not np.any(mask): + return fallback_flux_baseline() + + masked_model = model[mask] + masked_data = data[mask] + masked_weights = weights[mask] + denom = np.sum(masked_weights * masked_model ** 2) + + if not np.isfinite(denom) or denom <= 0: + ratio = masked_data / masked_model + ratio = ratio[np.isfinite(ratio)] + if ratio.size == 0: + return fallback_flux_baseline() + baseline = np.nanmedian(ratio) + return baseline if np.isfinite(baseline) else fallback_flux_baseline() + + baseline = np.sum(masked_weights * masked_data * masked_model) / denom + return baseline if np.isfinite(baseline) else fallback_flux_baseline() + + +def solve_flux_baseline_uncertainty(model, dataerr, mask=None): + if dataerr is None: + return 0.0 + model = np.asarray(model, dtype=float) + dataerr = np.asarray(dataerr, dtype=float) + fit_mask = normalized_optional_fit_mask(mask, model.shape) + mask = np.isfinite(model) & np.isfinite(dataerr) & (dataerr > 0) + if fit_mask is not None: + mask &= fit_mask + if not np.any(mask): + return 0.0 + denom = np.sum((model[mask] / dataerr[mask]) ** 2) + if not np.isfinite(denom) or denom <= 0: + return 0.0 + return (1.0 / denom) ** 0.5 + + +def mc_a1(m_a2, sig_a2, transit, airmass, data, dataerr=None, n=10000, mask=None): + n = int(n) a2 = np.random.normal(m_a2, sig_a2, n) - model = transit * np.exp(np.repeat(np.expand_dims(a2, 0), airmass.shape[0], 0).T * airmass) - detrend = data / model - return np.mean(np.median(detrend, 0)), np.std(np.median(detrend, 0)) + reference = get_airmass_reference(airmass) + transit = np.asarray(transit, dtype=float) + data = np.asarray(data, dtype=float) + airmass = np.asarray(airmass, dtype=float) + centered_airmass = center_airmass(airmass, reference=reference) + weights = np.ones(transit.shape[0], dtype=float) + fit_mask = normalized_optional_fit_mask(mask, transit.shape) + + if dataerr is not None: + dataerr = np.asarray(dataerr, dtype=float) + weights = np.zeros(transit.shape[0], dtype=float) + valid_err = np.isfinite(dataerr) & (dataerr > 0) + weights[valid_err] = 1.0 / (dataerr[valid_err] ** 2) + + mask = np.isfinite(data) & np.isfinite(transit) & np.isfinite(centered_airmass) + if fit_mask is not None: + mask &= fit_mask + if dataerr is not None: + mask &= np.isfinite(weights) & (weights > 0) + + if not np.any(mask): + return fallback_flux_baseline(), 0.0 + + masked_transit = transit[mask] + masked_airmass = centered_airmass[mask] + masked_data = data[mask] + masked_weights = weights[mask] + numer = np.empty(n, dtype=float) + denom = np.empty(n, dtype=float) + chunk_size = 1024 + + for start in range(0, n, chunk_size): + stop = min(start + chunk_size, n) + masked_model = masked_transit * np.exp(np.outer(a2[start:stop], masked_airmass)) + numer[start:stop] = np.sum(masked_weights * masked_data * masked_model, axis=1) + denom[start:stop] = np.sum(masked_weights * masked_model ** 2, axis=1) + valid = np.isfinite(numer) & np.isfinite(denom) & (denom > 0) + + if not np.any(valid): + best_model = transit * airmass_trend(m_a2, airmass, reference=reference) + baseline = solve_flux_baseline(best_model, data, dataerr, mask=fit_mask) + return baseline, solve_flux_baseline_uncertainty(best_model, dataerr, mask=fit_mask) + + baselines = numer[valid] / denom[valid] + baseline = float(np.nanmean(baselines)) + baseline_unc = float(np.nanstd(baselines)) + + if baseline_unc == 0.0: + best_model = transit * airmass_trend(m_a2, airmass, reference=reference) + baseline_unc = solve_flux_baseline_uncertainty(best_model, dataerr, mask=fit_mask) + + return baseline, baseline_unc def round_to_2(*args): @@ -118,6 +604,27 @@ def round_to_2(*args): return round(x, roundval) +def format_value_error_for_plot(value, error): + """Format value/error text with a two-significant-figure uncertainty.""" + try: + value = float(value) + except (TypeError, ValueError): + value = np.nan + + try: + error = float(error) + except (TypeError, ValueError): + error = np.nan + + if not np.isfinite(error) or error < 0: + value_text = f"{value:.6f}".rstrip('0').rstrip('.') if np.isfinite(value) else "n/a" + return value_text, "n/a" + + if not np.isfinite(value): + return "n/a", format_value_and_uncertainty(0, error)[1] + return format_value_and_uncertainty(value, error) + + # average data into bins of dt from start to finish def time_bin(time, flux, dt=1. / (60 * 24)): bins = int(np.floor((max(time) - min(time)) / dt)) @@ -127,9 +634,9 @@ def time_bin(time, flux, dt=1. / (60 * 24)): for i in range(bins): mask = (time >= (min(time) + i * dt)) & (time < (min(time) + (i + 1) * dt)) if mask.sum() > 0: - bflux[i] = np.nanmean(flux[mask]) - btime[i] = np.nanmean(time[mask]) - bstds[i] = np.nanstd(flux[mask]) / (mask.sum() ** 0.5) + bflux[i] = bn.nanmean(flux[mask]) + btime[i] = bn.nanmean(time[mask]) + bstds[i] = bn.nanstd(flux[mask]) / (mask.sum() ** 0.5) zmask = (bflux == 0) | (btime == 0) | np.isnan(bflux) | np.isnan(btime) return btime[~zmask], bflux[~zmask], bstds[~zmask] @@ -139,7 +646,7 @@ def binner(arr, n, err=''): if len(err) == 0: ecks = np.pad(arr.astype(float), (0, ((n - arr.size % n) % n)), mode='constant', constant_values=np.NaN).reshape(-1, n) - arr = np.nanmean(ecks, axis=1) + arr = bn.nanmean(ecks, axis=1) return arr else: ecks = np.pad(arr.astype(float), (0, ((n - arr.size % n) % n)), mode='constant', @@ -148,18 +655,65 @@ def binner(arr, n, err=''): -1, n) weights = 1. / (why ** 2.) # Calculate the weighted average - arr = np.nansum(ecks * weights, axis=1) / np.nansum(weights, axis=1) - err = np.array([np.sqrt(1. / np.nansum(1. / (np.array(i) ** 2.))) for i in why]) + arr = bn.nansum(ecks * weights, axis=1) / bn.nansum(weights, axis=1) + err = np.array([np.sqrt(1. / bn.nansum(1. / (np.array(i) ** 2.))) for i in why]) return arr, err +def normalize_exposure_times_seconds_to_days(exposure_times_seconds, reference_shape): + if exposure_times_seconds is None: + return None + try: + exposure_times_seconds = np.asarray(exposure_times_seconds, dtype=float) + except (TypeError, ValueError): + return None + + if exposure_times_seconds.shape == (): + exposure_times_seconds = np.full(reference_shape, float(exposure_times_seconds), dtype=float) + elif exposure_times_seconds.shape != reference_shape: + return None + else: + exposure_times_seconds = np.array(exposure_times_seconds, dtype=float, copy=True) + + valid = np.isfinite(exposure_times_seconds) & (exposure_times_seconds > MIN_EXPOSURE_SMEARING_SECONDS) + if not np.any(valid): + return None + + exposure_times_seconds[~valid] = 0.0 + return exposure_times_seconds / 86400.0 + + class lc_fitter(object): - def __init__(self, time, data, dataerr, airmass, prior, bounds, neighbors=200, mode='ns', jd_times=None, verbose=True): + def __init__( + self, + time, + data, + dataerr, + airmass, + prior, + bounds, + neighbors=200, + mode='ns', + jd_times=None, + verbose=True, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + keep_ultranest_sampler=False, + baseline_fit_mask=None, + fixed_parameter_errors=None, + fixed_flux_baseline=False, + ultranest_min_num_live_points=None, + exposure_times_seconds=None, + exposure_smearing_supersample=DEFAULT_EXPOSURE_SMEARING_SUPERSAMPLE, + exposure_smearing_change_tolerance=DEFAULT_EXPOSURE_SMEARING_CHANGE_TOLERANCE, + ultranest_warmstart_source=None, + ): self.time = time self.data = data self.dataerr = dataerr self.airmass = airmass + self.airmass_reference = get_airmass_reference(airmass) self.prior = prior self.bounds = bounds self.max_ncalls = 2e5 @@ -167,15 +721,3140 @@ def __init__(self, time, data, dataerr, airmass, prior, bounds, neighbors=200, m self.jd_times = jd_times self.mode = mode self.neighbors = neighbors + self.use_impactparameter_rather_than_inclination_to_fit = use_impactparameter_rather_than_inclination_to_fit + self.duration_prior = copy.deepcopy(duration_prior) if isinstance(duration_prior, dict) else None + self.keep_ultranest_sampler = bool(keep_ultranest_sampler) + self.baseline_fit_mask = self._coerce_baseline_fit_mask(baseline_fit_mask) + self.fixed_parameter_errors = ( + copy.deepcopy(fixed_parameter_errors) + if isinstance(fixed_parameter_errors, dict) + else {} + ) + self.fixed_flux_baseline = bool(fixed_flux_baseline) + self.ultranest_min_num_live_points = ultranest_min_num_live_points + self.ultranest_warmstart_source = ultranest_warmstart_source + self.ultranest_expanded_prior_warmstart_attempted = False + self.ultranest_expanded_prior_warmstart_applied = False + self.ultranest_expanded_prior_warmstart_note = None + self.ultranest_expanded_prior_warmstart_source_sample_count = 0 + self.ultranest_expanded_prior_warmstart_effective_sample_size = 0.0 + self.ultranest_expanded_prior_warmstart_expanded_keys = [] + self.ultranest_expanded_prior_warmstart_full_prior_fraction = ( + ULTRANEST_EXPANDED_PRIOR_WARMSTART_FULL_PRIOR_FRACTION + ) + self.exposure_times_days = normalize_exposure_times_seconds_to_days( + exposure_times_seconds, + np.asarray(time).shape, + ) + self.exposure_smearing_supersample = _exposure_smearing_supersample({ + EXPOSURE_SMEARING_SUPERSAMPLE_KEY: exposure_smearing_supersample, + }) + self.exposure_smearing_change_tolerance = _exposure_smearing_change_tolerance({ + EXPOSURE_SMEARING_CHANGE_TOLERANCE_KEY: exposure_smearing_change_tolerance, + }) + self.exposure_smearing_available = self.exposure_times_days is not None + if self.exposure_smearing_available: + finite_exposures = self.exposure_times_days[ + np.isfinite(self.exposure_times_days) & (self.exposure_times_days > 0) + ] + self.exposure_smearing_median_seconds = ( + float(np.nanmedian(finite_exposures) * 86400.0) + if finite_exposures.size + else np.nan + ) + else: + self.exposure_smearing_median_seconds = np.nan + self._ultranest_resume_context = None self.results = None + self.sampled_keys = list(bounds.keys()) + self.sample_bounds = copy.deepcopy(bounds) + self.impact_parameter_sampled_directly = False + self.sample_parameters = {} + self.sample_errors = {} + self.sample_quantiles = {} + self.nested_fit_fallback = False + self.nested_fit_failure_reason = None if self.mode == "lm": self.fit_LM() elif self.mode == "ns": - self.fit_nested() + try: + self.fit_nested() + except np.linalg.LinAlgError as exc: + self.nested_fit_fallback = True + self.nested_fit_failure_reason = f"{type(exc).__name__}: {exc}" + self.ns_type = 'lm' + self.mode = "lm" + if self.verbose: + print( + "WARNING: Nested light curve fitting failed with a linear algebra error; " + "falling back to least-squares fit." + ) + print(f" Reason: {self.nested_fit_failure_reason}") + self.fit_LM() + + def _validate_flux_baseline_keys(self): + free_flux_keys = [key for key in self.bounds if key in ('a0', 'a1')] + if len(free_flux_keys) > 1: + raise ValueError("Use only one of 'a0' or 'a1' as a free baseline parameter.") + + def _has_free_flux_baseline(self): + return has_explicit_flux_baseline(getattr(self, 'bounds', {})) + + def _uses_fixed_flux_baseline(self): + return bool(getattr(self, 'fixed_flux_baseline', False)) + + def _uses_analytic_flux_baseline(self): + return ( + np.ndim(getattr(self, 'airmass', np.array([]))) != 2 + and not self._has_free_flux_baseline() + and not self._uses_fixed_flux_baseline() + and hasattr(self, 'time') + and hasattr(self, 'data') + and hasattr(self, 'dataerr') + ) + + def _set_flux_baseline(self, value, error=0.0): + self.parameters['a0'] = value + self.errors['a0'] = error + self.parameters['a1'] = value + self.errors['a1'] = error + + def _values_with_analytic_flux_baseline(self, values): + values = copy.deepcopy(values) + if not self._uses_analytic_flux_baseline(): + return values + + try: + model = self._transit_model(self.time, values) + model = np.asarray(model, dtype=float) * airmass_trend( + values.get('a2', 0), + self.airmass, + reference=self._get_airmass_reference(), + ) + flux_scale = solve_flux_baseline( + model, + self.data, + self.dataerr, + mask=self._get_baseline_fit_mask(), + ) + except Exception: + return values + + values['a0'] = flux_scale + values['a1'] = flux_scale + return values + + def _coerce_baseline_fit_mask(self, baseline_fit_mask): + fit_mask = normalized_optional_fit_mask(baseline_fit_mask, np.asarray(self.time).shape) + if fit_mask is None: + return None + return fit_mask + + def _get_baseline_fit_mask(self): + return getattr(self, 'baseline_fit_mask', None) + + def _apply_fixed_parameter_errors(self): + fixed_errors = getattr(self, 'fixed_parameter_errors', None) + if not isinstance(fixed_errors, dict): + return + if not hasattr(self, 'parameters') or not isinstance(self.parameters, dict): + return + if not hasattr(self, 'errors') or not isinstance(self.errors, dict): + self.errors = {} + if not hasattr(self, 'quantiles') or not isinstance(self.quantiles, dict): + self.quantiles = {} + + for key, error in fixed_errors.items(): + if key not in self.parameters or key in self.errors: + continue + try: + error = float(error) + except (TypeError, ValueError): + continue + if not np.isfinite(error) or error < 0: + continue + self.errors[key] = error + self.quantiles[key] = [-error, error] + + def _get_airmass_reference(self): + if hasattr(self, 'airmass_reference'): + return self.airmass_reference + return get_airmass_reference(self.airmass) + + def _get_plot_time_range(self): + plot_time_range = normalize_time_range(getattr(self, 'plot_time_range', None)) + if plot_time_range is not None: + return plot_time_range + return normalize_time_range(self.time) + + def _exposure_times_for_model_times(self, times): + exposure_times_days = getattr(self, 'exposure_times_days', None) + if exposure_times_days is None: + return None + + times = np.asarray(times, dtype=float) + source_times = np.asarray(self.time, dtype=float) + if ( + times.shape == source_times.shape + and exposure_times_days.shape == source_times.shape + and np.allclose(times, source_times, rtol=0.0, atol=0.0) + ): + return exposure_times_days + + finite_exposures = exposure_times_days[ + np.isfinite(exposure_times_days) & (exposure_times_days > 0) + ] + if finite_exposures.size == 0: + return None + return float(np.nanmedian(finite_exposures)) + + def _values_with_exposure_smearing(self, values, times): + exposure_times_days = self._exposure_times_for_model_times(times) + if exposure_times_days is None: + return values + + smeared_values = copy.deepcopy(values) + smeared_values[EXPOSURE_SMEARING_EXPOSURE_TIME_KEY] = exposure_times_days + smeared_values[EXPOSURE_SMEARING_SUPERSAMPLE_KEY] = self.exposure_smearing_supersample + smeared_values[EXPOSURE_SMEARING_CHANGE_TOLERANCE_KEY] = self.exposure_smearing_change_tolerance + return smeared_values + + def _transit_model(self, times, values): + return transit(times, self._values_with_exposure_smearing(values, times)) + + def _update_plot_geometry(self): + plot_time_range = self._get_plot_time_range() + self.phase = get_plot_phase(self.time, self.parameters['per'], self.parameters['tmid'], plot_time_range) + + if plot_time_range is None: + self.time_upsample = np.linspace(min(self.time), max(self.time), 1000) + else: + self.time_upsample = np.linspace(plot_time_range[0], plot_time_range[1], 1000) + + self.transit_upsample = self._transit_model(self.time_upsample, self.parameters) + self.phase_upsample = get_plot_phase( + self.time_upsample, + self.parameters['per'], + self.parameters['tmid'], + plot_time_range, + ) + + def _build_systematics_model(self, values): + return get_flux_baseline(values) * airmass_trend( + values.get('a2', 0), + self.airmass, + reference=self._get_airmass_reference(), + ) + + def _build_systematics_model_at(self, values, times=None): + if times is None: + return self._build_systematics_model(values) + + times = np.asarray(times, dtype=float) + if np.ndim(self.airmass) == 2: + return np.full(times.shape, get_flux_baseline(values), dtype=float) + + source_times = np.asarray(self.time, dtype=float) + source_airmass = np.asarray(self.airmass, dtype=float) + finite = np.isfinite(source_times) & np.isfinite(source_airmass) + if times.shape == source_times.shape and np.allclose(times, source_times, rtol=0.0, atol=0.0): + airmass_values = source_airmass + elif np.count_nonzero(finite) >= 2: + order = np.argsort(source_times[finite]) + airmass_values = np.interp( + times, + source_times[finite][order], + source_airmass[finite][order], + left=source_airmass[finite][order][0], + right=source_airmass[finite][order][-1], + ) + elif np.count_nonzero(finite) == 1: + airmass_values = np.full(times.shape, source_airmass[finite][0], dtype=float) + else: + airmass_values = np.zeros(times.shape, dtype=float) + + return get_flux_baseline(values) * airmass_trend( + values.get('a2', 0), + airmass_values, + reference=self._get_airmass_reference(), + ) + + def _get_perturbed_transit_parameter_value(self, key, value): + try: + value = float(value) + except (TypeError, ValueError): + return np.nan + + if key in ('rprs', 'ars', 'per', 'a0', 'a1'): + return max(value, np.finfo(float).eps) + if key == 'ecc': + return float(np.clip(value, 0.0, 0.999999)) + if key == 'inc': + return float(np.clip(value, 0.0, 180.0)) + return value + + def _normalized_model_for_plot_times(self, times, values): + values = self._values_with_analytic_flux_baseline(values) + model = np.asarray(self._transit_model(times, values), dtype=float) + if np.ndim(self.airmass) == 2: + return model + + try: + sample_systematics = self._build_systematics_model_at(values, times) + best_systematics = self._build_systematics_model_at(self.parameters, times) + except Exception: + return model + + with np.errstate(divide='ignore', invalid='ignore'): + normalized_model = model * sample_systematics / best_systematics + if normalized_model.shape != model.shape or not np.any(np.isfinite(normalized_model)): + return model + return normalized_model + + def _posterior_model_uncertainty(self, times, sigma=1.0): + if getattr(self, 'results', None) is None: + return None + + try: + sample_points, sample_logl, sample_weights = self._get_triangle_plot_samples() + except Exception: + return None + + sample_points = np.asarray(sample_points, dtype=float) + if sample_points.ndim != 2 or sample_points.shape[0] < 2: + return None + + finite_rows = np.all(np.isfinite(sample_points), axis=1) + if sample_logl is not None: + sample_logl = np.asarray(sample_logl, dtype=float) + if sample_logl.shape[0] == sample_points.shape[0]: + finite_rows &= np.isfinite(sample_logl) + + if np.count_nonzero(finite_rows) < 2: + return None + + row_indices = np.flatnonzero(finite_rows) + if row_indices.size > MODEL_UNCERTAINTY_POSTERIOR_SAMPLE_LIMIT: + if sample_weights is not None: + weights_array = np.asarray(sample_weights, dtype=float) + if weights_array.shape[0] == sample_points.shape[0]: + row_weights = np.where(np.isfinite(weights_array[row_indices]), weights_array[row_indices], 0.0) + order = np.argsort(row_weights)[-MODEL_UNCERTAINTY_POSTERIOR_SAMPLE_LIMIT:] + row_indices = row_indices[np.sort(order)] + else: + row_indices = row_indices[ + np.linspace(0, row_indices.size - 1, MODEL_UNCERTAINTY_POSTERIOR_SAMPLE_LIMIT).astype(int) + ] + else: + row_indices = row_indices[ + np.linspace(0, row_indices.size - 1, MODEL_UNCERTAINTY_POSTERIOR_SAMPLE_LIMIT).astype(int) + ] + + selected_points = sample_points[row_indices] + selected_weights = None + if sample_weights is not None: + weights_array = np.asarray(sample_weights, dtype=float) + if weights_array.shape[0] == sample_points.shape[0]: + selected_weights = weights_array[row_indices] + selected_weights = np.where(np.isfinite(selected_weights) & (selected_weights >= 0), selected_weights, 0.0) + if np.sum(selected_weights) <= 0: + selected_weights = None + + bound_keys = list(self.bounds.keys()) + sampled_keys = getattr(self, 'sampled_keys', None) + if sampled_keys is None: + sampled_keys = self._get_sampled_keys(bound_keys) + models = [] + for point in selected_points: + try: + values = copy.deepcopy(self.parameters) + values.update(self._physical_values_from_sample_point(point, bound_keys, sampled_keys)) + model = self._normalized_model_for_plot_times(times, values) + except Exception: + continue + if model.shape == times.shape and np.all(np.isfinite(model)): + models.append(model) + + if len(models) < 2: + return None + + model_grid = np.asarray(models, dtype=float) + if selected_weights is not None and selected_weights.shape[0] != model_grid.shape[0]: + selected_weights = None + + try: + sigma = float(sigma) + except (TypeError, ValueError): + sigma = 1.0 + if not np.isfinite(sigma) or sigma <= 0: + sigma = 1.0 + coverage = math.erf(sigma / np.sqrt(2.0)) + q_lower = 0.5 * (1.0 - coverage) + q_upper = 1.0 - q_lower + + if selected_weights is None: + lower, median, upper = np.nanpercentile( + model_grid, + [100.0 * q_lower, 50.0, 100.0 * q_upper], + axis=0, + ) + else: + lower = np.array([ + self._weighted_quantiles(model_grid[:, i], [q_lower], weights=selected_weights)[0] + for i in range(model_grid.shape[1]) + ]) + median = np.array([ + self._weighted_quantiles(model_grid[:, i], [0.5], weights=selected_weights)[0] + for i in range(model_grid.shape[1]) + ]) + upper = np.array([ + self._weighted_quantiles(model_grid[:, i], [q_upper], weights=selected_weights)[0] + for i in range(model_grid.shape[1]) + ]) + + try: + best_model = self._normalized_model_for_plot_times(times, self.parameters) + except Exception: + best_model = median + + lower_width = median - lower + upper_width = upper - median + lower = best_model - np.maximum(lower_width, 0.0) + upper = best_model + np.maximum(upper_width, 0.0) + + finite = np.isfinite(lower) & np.isfinite(upper) & (lower <= upper) + if not np.any(finite): + return None + return lower, upper + + def transit_model_uncertainty(self, times=None, sigma=1.0): + if times is None: + times = getattr(self, 'time_upsample', self.time) + times = np.asarray(times, dtype=float) + if times.size == 0: + return None + + try: + model = self._normalized_model_for_plot_times(times, self.parameters) + except Exception: + return None + + posterior_envelope = self._posterior_model_uncertainty(times, sigma=sigma) + if posterior_envelope is not None: + return posterior_envelope + + sigma = float(sigma) + variance = np.zeros_like(model, dtype=float) + uncertainty_keys = list(TRANSIT_MODEL_UNCERTAINTY_KEYS) + for key in BASELINE_MODEL_UNCERTAINTY_KEYS: + if key == 'a1' and 'a0' in self.parameters: + continue + uncertainty_keys.append(key) + + for key in uncertainty_keys: + if key not in self.parameters: + continue + error = self.errors.get(key) + try: + center = float(self.parameters[key]) + error = float(error) + except (TypeError, ValueError): + continue + if not np.isfinite(center) or not np.isfinite(error) or error <= 0: + continue + + lower_value = self._get_perturbed_transit_parameter_value(key, center - error) + upper_value = self._get_perturbed_transit_parameter_value(key, center + error) + if ( + not np.isfinite(lower_value) + or not np.isfinite(upper_value) + or np.isclose(lower_value, upper_value) + ): + continue + + lower_parameters = copy.deepcopy(self.parameters) + upper_parameters = copy.deepcopy(self.parameters) + lower_parameters[key] = lower_value + upper_parameters[key] = upper_value + try: + lower_model = self._normalized_model_for_plot_times(times, lower_parameters) + upper_model = self._normalized_model_for_plot_times(times, upper_parameters) + except Exception: + continue + + derivative = (upper_model - lower_model) / (upper_value - lower_value) + contribution = derivative * error * sigma + finite = np.isfinite(contribution) + variance[finite] += contribution[finite] ** 2 + + model_uncertainty = np.sqrt(variance) + if not np.any(np.isfinite(model_uncertainty) & (model_uncertainty > 0)): + return None + return model - model_uncertainty, model + model_uncertainty + + def _empirical_uncertainty_value(self, key): + empirical_uncertainty = getattr(self, 'empirical_transit_uncertainty', None) + if not isinstance(empirical_uncertainty, dict) or not empirical_uncertainty.get('available'): + return np.nan + try: + value = float(empirical_uncertainty.get(key)) + except (TypeError, ValueError): + return np.nan + return value if np.isfinite(value) else np.nan + + def _empirical_baseline_uncertainty_fraction(self): + value = self._empirical_uncertainty_value('baseline_red_noise_uncertainty_fraction') + if not np.isfinite(value) or value <= 0: + value = self._empirical_uncertainty_value('depth_uncertainty_fraction') + return value if np.isfinite(value) and value > 0 else np.nan + + def _baseline_envelope_with_empirical_floor(self, envelope): + if envelope is None: + return None + + lower, upper = envelope + lower = np.asarray(lower, dtype=float) + upper = np.asarray(upper, dtype=float) + empirical_baseline_uncertainty = self._empirical_baseline_uncertainty_fraction() + if not np.isfinite(empirical_baseline_uncertainty) or empirical_baseline_uncertainty <= 0: + return lower, upper + + existing_width = np.maximum(1.0 - lower, upper - 1.0) + combined_width = np.sqrt( + np.where(np.isfinite(existing_width), existing_width, 0.0) ** 2 + + empirical_baseline_uncertainty ** 2 + ) + return 1.0 - combined_width, 1.0 + combined_width + + def _combined_rprs_uncertainty_for_reporting(self): + value = self._empirical_uncertainty_value('combined_rprs_uncertainty') + if np.isfinite(value) and value >= 0: + return value + try: + value = float(self.errors.get('rprs')) + except (TypeError, ValueError): + return np.nan + return value if np.isfinite(value) and value >= 0 else np.nan + + def _model_data_uncertainty_for_reporting(self, parameter_name): + try: + value = float(self.errors.get(parameter_name)) + except (TypeError, ValueError): + return np.nan + if not np.isfinite(value) or value < 0: + return np.nan + + beta = self._empirical_uncertainty_value('red_noise_beta_factor') + if not np.isfinite(beta) or beta < 1.0: + beta = 1.0 + return value * beta + + def _posterior_baseline_model_uncertainty(self, times, sigma=1.0): + if getattr(self, 'results', None) is None or np.ndim(getattr(self, 'airmass', np.array([]))) == 2: + return None + + try: + sample_points, sample_logl, sample_weights = self._get_triangle_plot_samples() + except Exception: + return None + + sample_points = np.asarray(sample_points, dtype=float) + if sample_points.ndim != 2 or sample_points.shape[0] < 2: + return None + + finite_rows = np.all(np.isfinite(sample_points), axis=1) + if sample_logl is not None: + sample_logl = np.asarray(sample_logl, dtype=float) + if sample_logl.shape[0] == sample_points.shape[0]: + finite_rows &= np.isfinite(sample_logl) + + if np.count_nonzero(finite_rows) < 2: + return None + + row_indices = np.flatnonzero(finite_rows) + if row_indices.size > MODEL_UNCERTAINTY_POSTERIOR_SAMPLE_LIMIT: + if sample_weights is not None: + weights_array = np.asarray(sample_weights, dtype=float) + if weights_array.shape[0] == sample_points.shape[0]: + row_weights = np.where(np.isfinite(weights_array[row_indices]), weights_array[row_indices], 0.0) + order = np.argsort(row_weights)[-MODEL_UNCERTAINTY_POSTERIOR_SAMPLE_LIMIT:] + row_indices = row_indices[np.sort(order)] + else: + row_indices = row_indices[ + np.linspace(0, row_indices.size - 1, MODEL_UNCERTAINTY_POSTERIOR_SAMPLE_LIMIT).astype(int) + ] + else: + row_indices = row_indices[ + np.linspace(0, row_indices.size - 1, MODEL_UNCERTAINTY_POSTERIOR_SAMPLE_LIMIT).astype(int) + ] + + selected_points = sample_points[row_indices] + selected_weights = None + if sample_weights is not None: + weights_array = np.asarray(sample_weights, dtype=float) + if weights_array.shape[0] == sample_points.shape[0]: + selected_weights = weights_array[row_indices] + selected_weights = np.where( + np.isfinite(selected_weights) & (selected_weights >= 0), + selected_weights, + 0.0, + ) + if np.sum(selected_weights) <= 0: + selected_weights = None + + try: + best_parameters = self._values_with_analytic_flux_baseline(self.parameters) + best_systematics = self._build_systematics_model_at(best_parameters, times) + except Exception: + return None + + best_systematics = np.asarray(best_systematics, dtype=float) + if best_systematics.shape != times.shape or not np.all(np.isfinite(best_systematics)): + return None + + bound_keys = list(getattr(self, 'bounds', {}).keys()) + sampled_keys = getattr(self, 'sampled_keys', None) + if sampled_keys is None: + sampled_keys = self._get_sampled_keys(bound_keys) + + ratios = [] + ratio_weights = [] + for index, point in enumerate(selected_points): + try: + values = copy.deepcopy(self.parameters) + values.update(self._physical_values_from_sample_point(point, bound_keys, sampled_keys)) + values = self._values_with_analytic_flux_baseline(values) + sample_systematics = self._build_systematics_model_at(values, times) + with np.errstate(divide='ignore', invalid='ignore'): + ratio = np.asarray(sample_systematics, dtype=float) / best_systematics + except Exception: + continue + if ratio.shape != times.shape or not np.all(np.isfinite(ratio)): + continue + ratios.append(ratio) + if selected_weights is not None: + ratio_weights.append(selected_weights[index]) + + if len(ratios) < 2: + return None + + ratio_grid = np.asarray(ratios, dtype=float) + if selected_weights is not None: + selected_weights = np.asarray(ratio_weights, dtype=float) + if selected_weights.shape[0] != ratio_grid.shape[0] or np.sum(selected_weights) <= 0: + selected_weights = None + + try: + sigma = float(sigma) + except (TypeError, ValueError): + sigma = 1.0 + if not np.isfinite(sigma) or sigma <= 0: + sigma = 1.0 + coverage = math.erf(sigma / np.sqrt(2.0)) + q_lower = 0.5 * (1.0 - coverage) + q_upper = 1.0 - q_lower + + if selected_weights is None: + lower, median, upper = np.nanpercentile( + ratio_grid, + [100.0 * q_lower, 50.0, 100.0 * q_upper], + axis=0, + ) + else: + lower = np.array([ + self._weighted_quantiles(ratio_grid[:, i], [q_lower], weights=selected_weights)[0] + for i in range(ratio_grid.shape[1]) + ]) + median = np.array([ + self._weighted_quantiles(ratio_grid[:, i], [0.5], weights=selected_weights)[0] + for i in range(ratio_grid.shape[1]) + ]) + upper = np.array([ + self._weighted_quantiles(ratio_grid[:, i], [q_upper], weights=selected_weights)[0] + for i in range(ratio_grid.shape[1]) + ]) + + lower_width = median - lower + upper_width = upper - median + lower = 1.0 - np.maximum(lower_width, 0.0) + upper = 1.0 + np.maximum(upper_width, 0.0) + finite = np.isfinite(lower) & np.isfinite(upper) & (lower <= upper) + if not np.any(finite): + return None + return lower, upper + + def baseline_model_uncertainty(self, times=None, sigma=1.0): + if times is None: + times = getattr(self, 'time_upsample', self.time) + times = np.asarray(times, dtype=float) + if times.size == 0 or np.ndim(getattr(self, 'airmass', np.array([]))) == 2: + return None + + posterior_envelope = self._posterior_baseline_model_uncertainty(times, sigma=sigma) + if posterior_envelope is not None: + return self._baseline_envelope_with_empirical_floor(posterior_envelope) + + try: + best_parameters = self._values_with_analytic_flux_baseline(self.parameters) + best_systematics = self._build_systematics_model_at(best_parameters, times) + except Exception: + return None + + if ( + np.asarray(best_systematics).shape != times.shape + or not np.any(np.isfinite(best_systematics)) + ): + return None + + try: + sigma = float(sigma) + except (TypeError, ValueError): + sigma = 1.0 + if not np.isfinite(sigma) or sigma <= 0: + sigma = 1.0 + + variance = np.zeros_like(times, dtype=float) + uses_analytic_flux_baseline = self._uses_analytic_flux_baseline() + a2_error = self.errors.get('a2') + try: + a2_error = float(a2_error) + except (TypeError, ValueError): + a2_error = 0.0 + for key in ('a0', 'a1', 'a2'): + if key == 'a1' and 'a0' in self.parameters: + continue + if key in ('a0', 'a1') and uses_analytic_flux_baseline and a2_error > 0: + continue + if key not in self.parameters: + continue + error = self.errors.get(key) + try: + center = float(self.parameters[key]) + error = float(error) + except (TypeError, ValueError): + continue + if not np.isfinite(center) or not np.isfinite(error) or error <= 0: + continue + + lower_value = self._get_perturbed_transit_parameter_value(key, center - error) + upper_value = self._get_perturbed_transit_parameter_value(key, center + error) + if ( + not np.isfinite(lower_value) + or not np.isfinite(upper_value) + or np.isclose(lower_value, upper_value) + ): + continue + + lower_parameters = copy.deepcopy(self.parameters) + upper_parameters = copy.deepcopy(self.parameters) + lower_parameters[key] = lower_value + upper_parameters[key] = upper_value + lower_parameters = self._values_with_analytic_flux_baseline(lower_parameters) + upper_parameters = self._values_with_analytic_flux_baseline(upper_parameters) + try: + lower_systematics = self._build_systematics_model_at(lower_parameters, times) + upper_systematics = self._build_systematics_model_at(upper_parameters, times) + except Exception: + continue + + with np.errstate(divide='ignore', invalid='ignore'): + lower_ratio = lower_systematics / best_systematics + upper_ratio = upper_systematics / best_systematics + derivative = (upper_ratio - lower_ratio) / (upper_value - lower_value) + contribution = derivative * error * sigma + finite = np.isfinite(contribution) + variance[finite] += contribution[finite] ** 2 + + baseline_uncertainty = np.sqrt(variance) + if not np.any(np.isfinite(baseline_uncertainty) & (baseline_uncertainty > 0)): + empirical_baseline_uncertainty = self._empirical_baseline_uncertainty_fraction() + if not np.isfinite(empirical_baseline_uncertainty) or empirical_baseline_uncertainty <= 0: + return None + return self._baseline_envelope_with_empirical_floor( + (1.0 - baseline_uncertainty, 1.0 + baseline_uncertainty) + ) + + def _plot_transit_model_uncertainty(self, ax, x_values, times, sort_index, label=None): + envelope = self.transit_model_uncertainty(times) + if envelope is None: + return None + + lower, upper = envelope + x_values = np.asarray(x_values, dtype=float) + sort_index = np.asarray(sort_index, dtype=int) + x_sorted = x_values[sort_index] + lower_sorted = np.asarray(lower, dtype=float)[sort_index] + upper_sorted = np.asarray(upper, dtype=float)[sort_index] + band = ax.fill_between( + x_sorted, + lower_sorted, + upper_sorted, + color='red', + alpha=0.16, + linewidth=0, + zorder=2.5, + label=label, + ) + ax.plot( + x_sorted, + lower_sorted, + color='red', + linestyle='--', + linewidth=0.9, + alpha=0.72, + zorder=3.4, + ) + ax.plot( + x_sorted, + upper_sorted, + color='red', + linestyle='--', + linewidth=0.9, + alpha=0.72, + zorder=3.4, + ) + return band + + def _plot_baseline_model_uncertainty(self, ax, x_values, times, sort_index, label=None): + envelope = self.baseline_model_uncertainty(times) + if envelope is None: + return None + + lower, upper = envelope + x_values = np.asarray(x_values, dtype=float) + sort_index = np.asarray(sort_index, dtype=int) + x_sorted = x_values[sort_index] + lower_sorted = np.asarray(lower, dtype=float)[sort_index] + upper_sorted = np.asarray(upper, dtype=float)[sort_index] + band = ax.fill_between( + x_sorted, + lower_sorted, + upper_sorted, + color='gold', + alpha=0.32, + linewidth=0, + zorder=2.1, + label=label, + ) + ax.plot( + x_sorted, + lower_sorted, + color='gold', + linestyle='--', + linewidth=0.9, + alpha=0.78, + zorder=3.2, + ) + ax.plot( + x_sorted, + upper_sorted, + color='gold', + linestyle='--', + linewidth=0.9, + alpha=0.78, + zorder=3.2, + ) + return band + + def _uses_internal_impact_parameter(self): + return ( + self.use_impactparameter_rather_than_inclination_to_fit + and self.mode == "ns" + and 'inc' in self.bounds + and 'b' not in self.bounds + ) + + def _get_sampled_keys(self, bound_keys=None): + bound_keys = list(self.bounds.keys()) if bound_keys is None else list(bound_keys) + if not self._uses_internal_impact_parameter(): + return bound_keys + return ['b' if key == 'inc' else key for key in bound_keys] + + def _get_impact_parameter_scale_upper_bound(self, values): + values = dict(values) + scale_keys = ('ars', 'ecc', 'omega') + endpoint_sets = [] + for key in scale_keys: + if key in self.bounds: + endpoints = np.asarray(self.bounds[key], dtype=float).reshape(-1)[:2] + else: + endpoints = np.asarray([values.get(key, 0.0)], dtype=float) + finite_endpoints = [float(value) for value in endpoints if np.isfinite(value)] + if not finite_endpoints: + return np.nan + endpoint_sets.append((key, finite_endpoints)) + + scales = [] + for candidate_values in product(*[endpoints for _, endpoints in endpoint_sets]): + candidate = dict(values) + for key, value in zip([key for key, _ in endpoint_sets], candidate_values): + candidate[key] = value + try: + scale = float(impact_parameter_scale(candidate)) + except (KeyError, TypeError, ValueError): + continue + if np.isfinite(scale) and scale > 0: + scales.append(scale) + + return float(max(scales)) if scales else np.nan + + def _get_impact_parameter_sampling_bounds(self, values=None, use_search_bounds=False): + values = self.prior if values is None else values + if use_search_bounds and 'rprs' in self.bounds: + values = dict(values) + rprs_bounds = np.asarray(self.bounds['rprs'], dtype=float).reshape(-1)[:2] + finite_rprs = rprs_bounds[np.isfinite(rprs_bounds) & (rprs_bounds >= 0)] + if finite_rprs.size > 0: + values['rprs'] = float(np.max(finite_rprs)) + + grazing_upper = grazing_impact_parameter(values) + if use_search_bounds: + scale_upper = self._get_impact_parameter_scale_upper_bound(values) + else: + try: + scale_upper = float(impact_parameter_scale(values)) + except (KeyError, TypeError, ValueError): + scale_upper = np.nan + + upper_candidates = [ + float(value) + for value in (grazing_upper, scale_upper) + if np.isfinite(value) and value > 0 + ] + upper = min(upper_candidates) if upper_candidates else 1.0 + return [0.0, float(max(0.0, upper))] + + def _get_impact_parameter_upper_bounds_for_sample_points(self, sample_points, bound_keys): + sample_points = np.atleast_2d(np.asarray(sample_points, dtype=float)) + bound_index = {key: index for index, key in enumerate(bound_keys)} + sample_count = sample_points.shape[0] + + def values_for(key, default): + if key in bound_index: + return sample_points[:, bound_index[key]] + value = np.asarray(self.prior.get(key, default), dtype=float) + if value.shape == (): + return np.full(sample_count, float(value), dtype=float) + return np.broadcast_to(value, (sample_count,)).astype(float) + + rprs = values_for('rprs', np.nan) + grazing_upper = np.where(np.isfinite(rprs) & (rprs >= 0), 1.0 + rprs, np.nan) + + ars = values_for('ars', np.nan) + ecc = values_for('ecc', 0.0) + omega = np.deg2rad(values_for('omega', 0.0)) + denom = 1.0 + ecc * np.sin(omega) + denom = np.where(np.isclose(denom, 0.0), np.finfo(float).eps, denom) + scale_upper = ars * (1.0 - ecc ** 2) / denom + + valid_grazing = np.isfinite(grazing_upper) & (grazing_upper > 0) + valid_scale = np.isfinite(scale_upper) & (scale_upper > 0) + upper = np.full(sample_count, 1.0, dtype=float) + + both_valid = valid_grazing & valid_scale + upper[both_valid] = np.minimum(grazing_upper[both_valid], scale_upper[both_valid]) + + grazing_only = valid_grazing & ~valid_scale + upper[grazing_only] = grazing_upper[grazing_only] + + scale_only = valid_scale & ~valid_grazing + upper[scale_only] = scale_upper[scale_only] + + return np.maximum(0.0, upper) + + def _get_sample_bounds(self, bound_keys=None, values=None): + bound_keys = list(self.bounds.keys()) if bound_keys is None else list(bound_keys) + sampled_keys = self._get_sampled_keys(bound_keys) + values = self.prior if values is None else values + sample_bounds = {} + for key, sampled_key in zip(bound_keys, sampled_keys): + if key == 'inc' and sampled_key == 'b': + sample_bounds[sampled_key] = self._get_impact_parameter_sampling_bounds( + values, + use_search_bounds=True, + ) + else: + sample_bounds[sampled_key] = list(self.bounds[key]) + return sample_bounds + + def _sample_point_from_unit_cube(self, upars, bound_keys=None): + bound_keys = list(self.bounds.keys()) if bound_keys is None else list(bound_keys) + upars_array = np.asarray(upars, dtype=float) + boundarray = np.array([self.bounds[k] for k in bound_keys], dtype=float) + lower_bounds = boundarray[:, 0] + bound_widths = boundarray[:, 1] - lower_bounds + uses_internal_impact_parameter = self._uses_internal_impact_parameter() + inc_indices = [i for i, key in enumerate(bound_keys) if key == 'inc'] + + sample_point = lower_bounds + bound_widths * upars_array + if not uses_internal_impact_parameter or not inc_indices: + return sample_point + + if len(inc_indices) == 1: + inc_index = inc_indices[0] + if upars_array.ndim == 2: + upper_bounds = self._get_impact_parameter_upper_bounds_for_sample_points( + sample_point, + bound_keys, + ) + sample_point[:, inc_index] = upper_bounds * upars_array[:, inc_index] + return sample_point + + upper_bound = self._get_impact_parameter_upper_bounds_for_sample_points( + sample_point.reshape(1, -1), + bound_keys, + )[0] + sample_point[inc_index] = upper_bound * upars_array[inc_index] + return sample_point + + if upars_array.ndim == 2: + for row_index, row_sample_point in enumerate(sample_point): + physical = dict(self.prior) + for i, key in enumerate(bound_keys): + if i not in inc_indices: + physical[key] = row_sample_point[i] + for i in inc_indices: + b_lower, b_upper = self._get_impact_parameter_sampling_bounds(physical) + row_sample_point[i] = b_lower + (b_upper - b_lower) * upars_array[row_index, i] + return sample_point + + physical = dict(self.prior) + for i, key in enumerate(bound_keys): + if i not in inc_indices: + physical[key] = sample_point[i] + for i in inc_indices: + b_lower, b_upper = self._get_impact_parameter_sampling_bounds(physical) + sample_point[i] = b_lower + (b_upper - b_lower) * upars_array[i] + + return sample_point + + def _unit_cube_from_sample_points(self, sample_points, bound_keys=None): + bound_keys = list(self.bounds.keys()) if bound_keys is None else list(bound_keys) + points = np.asarray(sample_points, dtype=float) + scalar_input = points.ndim == 1 + points_2d = np.atleast_2d(points) + if points_2d.shape[1] != len(bound_keys): + raise ValueError( + "Sample-point dimensionality does not match the expanded-prior bounds." + ) + + boundarray = np.array([self.bounds[key] for key in bound_keys], dtype=float) + widths = boundarray[:, 1] - boundarray[:, 0] + if ( + not np.all(np.isfinite(boundarray)) + or not np.all(np.isfinite(widths)) + or np.any(widths <= 0) + ): + raise ValueError("Expanded-prior bounds are not finite and increasing.") + + unit_points = (points_2d - boundarray[:, 0]) / widths + if self._uses_internal_impact_parameter() and 'inc' in bound_keys: + inc_index = bound_keys.index('inc') + upper_bounds = self._get_impact_parameter_upper_bounds_for_sample_points( + points_2d, + bound_keys, + ) + valid_upper = np.isfinite(upper_bounds) & (upper_bounds > 0) + if not np.all(valid_upper): + raise ValueError( + "Impact-parameter upper bounds are invalid for warm-start samples." + ) + unit_points[:, inc_index] = points_2d[:, inc_index] / upper_bounds + + return unit_points[0] if scalar_input else unit_points + + @staticmethod + def _warmstart_values_match(left, right): + if left is None or right is None: + return left is None and right is None + if isinstance(left, dict) or isinstance(right, dict): + if not isinstance(left, dict) or not isinstance(right, dict): + return False + if set(left) != set(right): + return False + return all( + lc_fitter._warmstart_values_match(left[key], right[key]) + for key in left + ) + try: + left_array = np.asarray(left) + right_array = np.asarray(right) + if left_array.shape != right_array.shape: + return False + if ( + np.issubdtype(left_array.dtype, np.number) + and np.issubdtype(right_array.dtype, np.number) + ): + return bool(np.array_equal( + left_array.astype(float), + right_array.astype(float), + equal_nan=True, + )) + return bool(np.array_equal(left_array, right_array)) + except (TypeError, ValueError): + return left == right + + def _expanded_prior_warmstart_compatibility(self, source, bound_keys, sampled_keys): + if source is None: + return None, "No previous UltraNest fit was supplied." + if getattr(source, 'ns_type', None) != 'ultranest': + return None, "The previous fit is not an UltraNest result." + + source_bound_keys = list(getattr(source, 'bounds', {}).keys()) + source_sampled_keys = list(getattr(source, 'sampled_keys', [])) + if source_bound_keys != list(bound_keys) or source_sampled_keys != list(sampled_keys): + return None, "The sampled parameterization changed between UltraNest fits." + + expanded_keys = [] + tolerance = 1e-12 + for key in bound_keys: + try: + source_lower, source_upper = [ + float(value) + for value in np.asarray(source.bounds[key], dtype=float).reshape(-1)[:2] + ] + target_lower, target_upper = [ + float(value) + for value in np.asarray(self.bounds[key], dtype=float).reshape(-1)[:2] + ] + except (KeyError, TypeError, ValueError): + return None, f"The {key} bounds are unavailable for warm-start validation." + if ( + target_lower > source_lower + tolerance + or target_upper < source_upper - tolerance + ): + return None, f"The {key} bounds contracted instead of forming a true superset." + if ( + target_lower < source_lower - tolerance + or target_upper > source_upper + tolerance + ): + expanded_keys.append(key) + + if not expanded_keys: + return None, "The prior bounds did not expand." + + likelihood_attributes = ( + 'time', + 'data', + 'dataerr', + 'airmass', + 'exposure_times_days', + 'baseline_fit_mask', + 'duration_prior', + 'fixed_flux_baseline', + 'use_impactparameter_rather_than_inclination_to_fit', + ) + for attribute_name in likelihood_attributes: + if not self._warmstart_values_match( + getattr(source, attribute_name, None), + getattr(self, attribute_name, None), + ): + return None, ( + f"The likelihood input {attribute_name} changed between UltraNest fits." + ) + + free_keys = set(bound_keys) + source_prior = getattr(source, 'prior', {}) + if not isinstance(source_prior, dict) or not isinstance(self.prior, dict): + return None, "The fixed model parameters are unavailable for warm-start validation." + fixed_keys = (set(source_prior) | set(self.prior)) - free_keys + for key in fixed_keys: + if not self._warmstart_values_match( + source_prior.get(key), + self.prior.get(key), + ): + return None, f"The fixed model parameter {key} changed between UltraNest fits." + + try: + weighted_samples = source.results['weighted_samples'] + source_points = np.asarray(weighted_samples['points'], dtype=float) + source_weights = np.asarray(weighted_samples['weights'], dtype=float) + except (AttributeError, KeyError, TypeError, ValueError): + return None, "The previous weighted posterior samples are unavailable." + + parameter_count = len(sampled_keys) + if ( + source_points.ndim != 2 + or source_points.shape[1] < parameter_count + or source_weights.ndim != 1 + or source_weights.shape[0] != source_points.shape[0] + ): + return None, "The previous weighted posterior sample arrays are malformed." + + source_points = source_points[:, :parameter_count] + valid = ( + np.all(np.isfinite(source_points), axis=1) + & np.isfinite(source_weights) + & (source_weights > 0) + ) + source_points = source_points[valid] + source_weights = source_weights[valid] + minimum_count = max( + ULTRANEST_EXPANDED_PRIOR_WARMSTART_MINIMUM_SAMPLE_COUNT, + 4 * max(1, parameter_count), + ) + if source_points.shape[0] < minimum_count: + return None, ( + f"Only {source_points.shape[0]} valid previous posterior samples are available; " + f"at least {minimum_count} are required." + ) + + weight_sum = float(np.sum(source_weights)) + if not np.isfinite(weight_sum) or weight_sum <= 0: + return None, "The previous posterior sample weights are invalid." + source_weights = source_weights / weight_sum + effective_sample_size = float(1.0 / np.sum(source_weights ** 2)) + minimum_effective_count = max(16, 2 * max(1, parameter_count)) + if not np.isfinite(effective_sample_size) or effective_sample_size < minimum_effective_count: + return None, ( + f"The previous posterior effective sample size is only " + f"{effective_sample_size:.1f}; at least {minimum_effective_count} is required." + ) + + return { + 'expanded_keys': expanded_keys, + 'points': source_points, + 'weights': source_weights, + 'effective_sample_size': effective_sample_size, + }, None + + @staticmethod + def _bounded_warmstart_posterior_sample(points, weights, maximum_count): + if points.shape[0] <= maximum_count: + return points, weights + + cumulative = np.cumsum(weights) + cumulative[-1] = 1.0 + quantiles = (np.arange(maximum_count, dtype=float) + 0.5) / maximum_count + selected = np.searchsorted(cumulative, quantiles, side='left') + selected = np.clip(selected, 0, points.shape[0] - 1) + return points[selected], np.full(maximum_count, 1.0 / maximum_count) + + def _build_expanded_prior_warmstart_problem( + self, + bound_keys, + sampled_keys, + loglike, + prior_transform, + ): + source = getattr(self, 'ultranest_warmstart_source', None) + compatibility, reason = self._expanded_prior_warmstart_compatibility( + source, + bound_keys, + sampled_keys, + ) + if compatibility is None: + return None, reason + + points, weights = self._bounded_warmstart_posterior_sample( + compatibility['points'], + compatibility['weights'], + ULTRANEST_EXPANDED_PRIOR_WARMSTART_MAXIMUM_SAMPLE_COUNT, + ) + unit_points = np.asarray( + self._unit_cube_from_sample_points(points, bound_keys), + dtype=float, + ) + tolerance = 1e-10 + inside = ( + np.all(np.isfinite(unit_points), axis=1) + & np.all(unit_points >= -tolerance, axis=1) + & np.all(unit_points <= 1.0 + tolerance, axis=1) + ) + unit_points = unit_points[inside] + weights = weights[inside] + if unit_points.shape[0] < ULTRANEST_EXPANDED_PRIOR_WARMSTART_MINIMUM_SAMPLE_COUNT: + return None, ( + "Too few previous posterior samples remain inside the expanded prior." + ) + + weights = weights / np.sum(weights) + unit_points = np.clip(unit_points, 1e-12, 1.0 - 1e-12) + full_prior_fraction = float(np.clip( + ULTRANEST_EXPANDED_PRIOR_WARMSTART_FULL_PRIOR_FRACTION, + 1e-6, + 1.0 - 1e-6, + )) + parameter_count = len(sampled_keys) + weighted_mean = np.sum(unit_points * weights[:, None], axis=0) + weighted_variance = np.sum( + weights[:, None] * (unit_points - weighted_mean) ** 2, + axis=0, + ) + # A modest scale floor avoids a singular hot proposal when the old + # posterior is extremely narrow. The 50% uniform component below is + # the stronger defense and guarantees full expanded-prior support. + hot_scale = np.clip(np.sqrt(np.maximum(weighted_variance, 0.0)), 0.02, 0.5) + hot_mean = np.clip(weighted_mean, 1e-8, 1.0 - 1e-8) + hot_alpha = -hot_mean / hot_scale + hot_beta = (1.0 - hot_mean) / hot_scale + hot_cdf_lower = ndtr(hot_alpha) + hot_cdf_width = np.maximum( + ndtr(hot_beta) - hot_cdf_lower, + np.finfo(float).tiny, + ) + log_hot_normalization = np.log(hot_cdf_width) + log_two_pi_half = 0.5 * np.log(2.0 * np.pi) + log_full_fraction = np.log(full_prior_fraction) + log_hot_fraction = np.log1p(-full_prior_fraction) + + def hot_transform(unit_values): + probabilities = hot_cdf_lower + unit_values * hot_cdf_width + probabilities = np.clip( + probabilities, + np.finfo(float).eps, + 1.0 - np.finfo(float).eps, + ) + return np.clip( + hot_mean + hot_scale * ndtri(probabilities), + 0.0, + 1.0, + ) + + def hot_log_density(unit_values): + standardized = (unit_values - hot_mean) / hot_scale + return np.sum( + -0.5 * standardized ** 2 + - np.log(hot_scale) + - log_two_pi_half + - log_hot_normalization, + axis=1, + ) + + # Defensive importance proposal: + # + # q_mix(u) = f * Uniform(u) + (1-f) * q_hot(u) + # + # The latent selector samples one of those components, while every + # point receives the common correction pi(u)/q_mix(u). Unlike using a + # branch-specific correction, this remains evidence-correct even if + # nested sampling prunes the component that contributes negligibly in + # a particular likelihood region. Half of all proposal mass still + # comes directly from the complete expanded prior. + def defensive_transform(unit_values): + values = np.asarray(unit_values, dtype=float) + scalar_input = values.ndim == 1 + values_2d = np.atleast_2d(values) + if values_2d.shape[1] != parameter_count + 1: + raise ValueError( + "Expanded-prior warm-start unit points have the wrong dimensionality." + ) + + proposal_unit = np.empty( + (values_2d.shape[0], parameter_count), + dtype=float, + ) + full_prior_mask = values_2d[:, -1] < full_prior_fraction + proposal_unit[full_prior_mask] = values_2d[ + full_prior_mask, :parameter_count + ] + hot_mask = ~full_prior_mask + if np.any(hot_mask): + proposal_unit[hot_mask] = hot_transform( + values_2d[hot_mask, :parameter_count] + ) + + log_q_hot = hot_log_density(proposal_unit) + log_q_mix = np.logaddexp( + log_full_fraction, + log_hot_fraction + log_q_hot, + ) + transformed = np.empty( + (values_2d.shape[0], parameter_count + 1), + dtype=float, + ) + transformed[:, :parameter_count] = prior_transform(proposal_unit) + transformed[:, -1] = -log_q_mix + + return transformed[0] if scalar_input else transformed + + def defensive_loglike(parameters): + values = np.asarray(parameters, dtype=float) + physical = values[..., :parameter_count] + correction = values[..., parameter_count] + return loglike(physical) + correction + + self.ultranest_expanded_prior_warmstart_source_sample_count = int( + unit_points.shape[0] + ) + self.ultranest_expanded_prior_warmstart_effective_sample_size = float( + compatibility['effective_sample_size'] + ) + self.ultranest_expanded_prior_warmstart_expanded_keys = list( + compatibility['expanded_keys'] + ) + return { + 'param_names': list(sampled_keys) + ['aux_logweight'], + 'loglike': defensive_loglike, + 'transform': defensive_transform, + 'vectorized': True, + 'full_prior_fraction': full_prior_fraction, + }, None + + @staticmethod + def _expanded_prior_warmstart_result_is_usable(results, parameter_count): + try: + maximum_likelihood = np.asarray( + results['maximum_likelihood']['point'], + dtype=float, + ) + weighted_points = np.asarray( + results['weighted_samples']['points'], + dtype=float, + ) + weighted_logl = np.asarray( + results['weighted_samples']['logl'], + dtype=float, + ) + except (KeyError, TypeError, ValueError): + return False + return bool( + maximum_likelihood.ndim == 1 + and maximum_likelihood.size >= parameter_count + and np.all(np.isfinite(maximum_likelihood[:parameter_count])) + and weighted_points.ndim == 2 + and weighted_points.shape[0] > 0 + and weighted_points.shape[1] >= parameter_count + and weighted_logl.shape == (weighted_points.shape[0],) + and np.any(np.isfinite(weighted_logl)) + ) + + @staticmethod + def _restore_expanded_prior_physical_likelihoods(results, loglike, parameter_count): + weighted_samples = results['weighted_samples'] + points = np.asarray(weighted_samples['points'], dtype=float) + auxiliary_logl = np.asarray(weighted_samples['logl'], dtype=float) + physical_logl = np.asarray( + loglike(points[:, :parameter_count]), + dtype=float, + ) + if physical_logl.shape != (points.shape[0],) or not np.any(np.isfinite(physical_logl)): + raise ValueError( + "the corrected warm-start samples could not be evaluated under the physical likelihood" + ) + + weighted_samples['auxiliary_logl'] = auxiliary_logl.copy() + weighted_samples['logl'] = physical_logl + if points.shape[1] > parameter_count: + weighted_samples['auxiliary_points'] = points[:, parameter_count:].copy() + weighted_samples['points'] = points[:, :parameter_count].copy() + maximum_index = int(np.nanargmax(physical_logl)) + maximum_likelihood = results.setdefault('maximum_likelihood', {}) + # The auxiliary likelihood contains a proposal correction and is not + # the physical maximum-likelihood criterion reported by EXOTIC. + maximum_likelihood['auxiliary_point'] = points[maximum_index].copy() + maximum_likelihood['point'] = points[maximum_index, :parameter_count].copy() + maximum_likelihood['logl'] = float(physical_logl[maximum_index]) + + equal_weight_samples = np.asarray(results.get('samples'), dtype=float) + if ( + equal_weight_samples.ndim == 2 + and equal_weight_samples.shape[1] >= parameter_count + ): + if equal_weight_samples.shape[1] > parameter_count: + results['auxiliary_samples'] = equal_weight_samples[:, parameter_count:].copy() + results['samples'] = equal_weight_samples[:, :parameter_count].copy() + + posterior = results.get('posterior') + if isinstance(posterior, dict): + for key, value in list(posterior.items()): + value_array = np.asarray(value) + if value_array.ndim == 1 and value_array.size == points.shape[1]: + posterior[key] = value_array[:parameter_count].copy() + + def _physical_values_from_sample_point(self, sample_point, bound_keys=None, sampled_keys=None): + bound_keys = list(self.bounds.keys()) if bound_keys is None else list(bound_keys) + sampled_keys = self._get_sampled_keys(bound_keys) if sampled_keys is None else list(sampled_keys) + physical = dict(self.prior) + impact_parameter = None + + for value, bound_key, sampled_key in zip(sample_point, bound_keys, sampled_keys): + if sampled_key == 'b' and bound_key == 'inc': + impact_parameter = value + continue + physical[bound_key] = value + + if impact_parameter is not None: + physical['b'] = impact_parameter + physical['inc'] = float(inclination_from_impact_parameter(physical, impact_parameter)) + + return physical + + def _summarize_derived_parameter(self, samples, point_estimate): + samples = np.asarray(samples, dtype=float) + center = float(point_estimate) + std = float(np.nanstd(samples)) + lower = float(np.nanpercentile(samples, 16)) + upper = float(np.nanpercentile(samples, 84)) + return center, std, [lower - center, upper - center] + + def _get_ultranest_weighted_sample_arrays(self): + try: + weighted_samples = self.results['weighted_samples'] + points = np.asarray(weighted_samples['points'], dtype=float) + logl = np.asarray(weighted_samples['logl'], dtype=float) + except Exception: + return None, None + + if points.ndim != 2 or points.shape[0] == 0: + return None, None + if logl.shape[0] != points.shape[0]: + return None, None + sampled_key_count = len(getattr(self, 'sampled_keys', [])) + if sampled_key_count and points.shape[1] >= sampled_key_count: + points = points[:, :sampled_key_count] + return points, logl + + def _loglike_neighborhood_uncertainty(self, parameter_index, center, minimum_count=8, points=None, logl=None): + if points is None or logl is None: + points, logl = self._get_ultranest_weighted_sample_arrays() + if points is None or parameter_index >= points.shape[1]: + return None + + values = np.asarray(points[:, parameter_index], dtype=float) + finite = np.isfinite(values) & np.isfinite(logl) + if np.count_nonzero(finite) < 2: + return None + + finite_values = values[finite] + finite_logl = logl[finite] + max_logl = float(np.nanmax(finite_logl)) + if not np.isfinite(max_logl): + return None + + selected_values = None + selected_delta = np.inf + for delta_chi2 in (1.0, 4.0, 9.0, 16.0, 25.0, np.inf): + if np.isfinite(delta_chi2): + mask = 2.0 * (max_logl - finite_logl) <= delta_chi2 + else: + mask = np.ones(finite_logl.shape, dtype=bool) + if np.count_nonzero(mask) >= minimum_count or delta_chi2 == np.inf: + selected_values = finite_values[mask] + selected_delta = delta_chi2 + break + + if selected_values is None or selected_values.size < 2: + return None + + lower, upper = np.nanpercentile(selected_values, [15.8655, 84.1345]) + std = float(np.nanstd(selected_values)) + half_width = float(0.5 * (upper - lower)) + candidates = [value for value in (std, half_width) if np.isfinite(value) and value > 0] + if not candidates: + return None + + error = float(max(candidates)) + return { + 'error': error, + 'quantiles': [float(lower), float(upper)], + 'sample_count': int(selected_values.size), + 'delta_chi2': float(selected_delta), + } + + def _ultranest_error_needs_sample_fallback(self, parameter_index, center, reported_error, points=None): + if points is None: + points, _ = self._get_ultranest_weighted_sample_arrays() + if points is None or parameter_index >= points.shape[1]: + return False + + values = np.asarray(points[:, parameter_index], dtype=float) + finite_values = values[np.isfinite(values)] + if finite_values.size < 2: + return False + + sample_scale = float(np.nanstd(finite_values)) + if not np.isfinite(sample_scale) or sample_scale <= 0: + return False + + try: + reported_error = float(reported_error) + except (TypeError, ValueError): + return True + + if not np.isfinite(reported_error) or reported_error <= 0: + return True + + absolute_floor = max(abs(float(center)) * 1e-12, np.finfo(float).eps) + return reported_error <= absolute_floor or reported_error < sample_scale * 1e-6 + + def _ultranest_error_is_inflated_relative_to_local_fit(self, reported_error, local_uncertainty): + if not isinstance(local_uncertainty, dict): + return False + + try: + reported_error = float(reported_error) + local_error = float(local_uncertainty.get('error', np.nan)) + delta_chi2 = float(local_uncertainty.get('delta_chi2', np.inf)) + except (TypeError, ValueError): + return False + + if ( + not np.isfinite(reported_error) + or reported_error <= 0 + or not np.isfinite(local_error) + or local_error <= 0 + ): + return False + if not np.isfinite(delta_chi2) or delta_chi2 > ULTRANEST_LOCAL_UNCERTAINTY_MAX_DELTA_CHI2: + return False + + return reported_error > local_error * ULTRANEST_INFLATED_ERROR_REPLACEMENT_FACTOR + + def _get_plot_range(self, key): + sample_parameters = getattr(self, 'sample_parameters', {}) + sample_errors = getattr(self, 'sample_errors', {}) + sample_bounds = getattr(self, 'sample_bounds', getattr(self, 'bounds', {})) + center = sample_parameters[key] if key in sample_parameters else self.parameters[key] + error = sample_errors[key] if key in sample_errors else self.errors[key] + + if isinstance(sample_bounds, dict) and key in sample_bounds: + try: + lower, upper = [ + float(value) for value in np.asarray(sample_bounds[key], dtype=float).reshape(-1)[:2] + ] + except (TypeError, ValueError, IndexError): + lower = np.nan + upper = np.nan + if np.isfinite(lower) and np.isfinite(upper) and lower < upper: + return [lower, upper] + + lower = center - 5 * error + upper = center + 5 * error + if np.isfinite(lower) and np.isfinite(upper) and lower < upper: + return [lower, upper] + + pad = error if np.isfinite(error) and error > 0 else max(abs(center) * 1e-6, 1e-6) + return [center - pad, center + pad] + + def _expand_plot_range_for_sample_cloud( + self, + key, + plot_range, + sample_values, + center, + required_visible_fraction=1.0, + ): + sample_values = np.asarray(sample_values, dtype=float) + finite_values = sample_values[np.isfinite(sample_values)] + if finite_values.size < 2: + return plot_range + + lower, upper = [float(value) for value in plot_range] + in_range = (finite_values >= lower) & (finite_values <= upper) + visible_fraction = np.count_nonzero(in_range) / float(finite_values.size) + if visible_fraction >= required_visible_fraction: + return plot_range + + new_lower = min(float(np.nanmin(finite_values)), float(center)) + new_upper = max(float(np.nanmax(finite_values)), float(center)) + padding = 0.05 * (new_upper - new_lower) + if not np.isfinite(padding) or padding <= 0: + padding = max(abs(float(center)) * 1e-6, 1e-6) + new_lower -= padding + new_upper += padding + + sample_bounds = getattr(self, 'sample_bounds', self.bounds) + if key in sample_bounds: + bound_lower, bound_upper = sample_bounds[key] + new_lower = max(new_lower, float(bound_lower)) + new_upper = min(new_upper, float(bound_upper)) + + if not np.isfinite(new_lower) or not np.isfinite(new_upper) or new_lower >= new_upper: + return plot_range + return [float(new_lower), float(new_upper)] + + def _histogram_edge_peak_fractions(self, sample_values, plot_range, bins, weights=None): + sample_values = np.asarray(sample_values, dtype=float) + finite_mask = np.isfinite(sample_values) + finite_weights = None + + if weights is not None: + weights = np.asarray(weights, dtype=float) + if weights.shape == sample_values.shape: + finite_mask &= np.isfinite(weights) & (weights >= 0) + finite_weights = weights[finite_mask] + finite_weight_sum = np.sum(finite_weights) + if ( + finite_weights.size == 0 + or not np.isfinite(finite_weight_sum) + or finite_weight_sum <= 0 + ): + finite_weights = None + + finite_values = sample_values[finite_mask] + if finite_values.size < 2: + return np.nan, np.nan, np.nan + + try: + lower, upper = [float(value) for value in np.asarray(plot_range, dtype=float).reshape(-1)[:2]] + except (TypeError, ValueError, IndexError): + return np.nan, np.nan, np.nan + + if not np.isfinite(lower) or not np.isfinite(upper) or lower >= upper: + return np.nan, np.nan, np.nan + + bins = max(1, int(bins)) + counts, _ = np.histogram( + finite_values, + bins=bins, + range=(lower, upper), + weights=finite_weights, + ) + counts = np.asarray(counts, dtype=float) + if counts.size == 0: + return np.nan, np.nan, np.nan + + peak = float(np.nanmax(counts)) + if not np.isfinite(peak) or peak <= 0: + return np.nan, np.nan, np.nan + + lower_fraction = float(counts[0] / peak) + upper_fraction = float(counts[-1] / peak) + return lower_fraction, upper_fraction, peak + + def _get_plot_range_expansion_bounds(self, key, sample_values, center): + sample_bounds = getattr(self, 'sample_bounds', getattr(self, 'bounds', {})) + if isinstance(sample_bounds, dict) and key in sample_bounds: + try: + bound_lower, bound_upper = [ + float(value) for value in np.asarray(sample_bounds[key], dtype=float).reshape(-1)[:2] + ] + except (TypeError, ValueError, IndexError): + bound_lower = np.nan + bound_upper = np.nan + + if np.isfinite(bound_lower) and np.isfinite(bound_upper) and bound_lower < bound_upper: + fallback_bounds = TRIANGLE_PLOT_FALLBACK_EXPANSION_BOUNDS.get(key) + if fallback_bounds is not None: + fallback_lower, fallback_upper = fallback_bounds + if ( + np.isfinite(fallback_lower) + and np.isfinite(fallback_upper) + and fallback_lower < fallback_upper + ): + return [ + float(min(bound_lower, fallback_lower)), + float(max(bound_upper, fallback_upper)), + ] + return [bound_lower, bound_upper] + + fallback_bounds = TRIANGLE_PLOT_FALLBACK_EXPANSION_BOUNDS.get(key) + if fallback_bounds is not None: + fallback_lower, fallback_upper = fallback_bounds + if np.isfinite(fallback_lower) and np.isfinite(fallback_upper) and fallback_lower < fallback_upper: + return [float(fallback_lower), float(fallback_upper)] + + sample_values = np.asarray(sample_values, dtype=float) + finite_values = sample_values[np.isfinite(sample_values)] + try: + center = float(center) + except (TypeError, ValueError): + center = np.nan + if np.isfinite(center): + finite_values = np.concatenate([finite_values, [center]]) + if finite_values.size < 2: + return None + + bound_lower = float(np.nanmin(finite_values)) + bound_upper = float(np.nanmax(finite_values)) + width = bound_upper - bound_lower + if not np.isfinite(width) or width <= 0: + padding = max(abs(float(center)) * 1e-6 if np.isfinite(center) else 0.0, 1e-6) + else: + padding = 0.05 * width + return [bound_lower - padding, bound_upper + padding] + + def _expand_plot_range_for_histogram_edge_dropoff( + self, + key, + plot_range, + sample_values, + center, + bins=None, + weights=None, + max_edge_peak_fraction=TRIANGLE_PLOT_EDGE_PEAK_FRACTION_MAX, + minimum_count=TRIANGLE_PLOT_EDGE_MIN_SAMPLE_COUNT, + max_steps=TRIANGLE_PLOT_EDGE_EXPANSION_STEPS, + ): + sample_values = np.asarray(sample_values, dtype=float) + finite_values = sample_values[np.isfinite(sample_values)] + if finite_values.size < int(minimum_count): + return plot_range + + try: + lower, upper = [float(value) for value in np.asarray(plot_range, dtype=float).reshape(-1)[:2]] + except (TypeError, ValueError, IndexError): + return plot_range + + if not np.isfinite(lower) or not np.isfinite(upper) or lower >= upper: + return plot_range + + expansion_bounds = self._get_plot_range_expansion_bounds(key, finite_values, center) + if expansion_bounds is None: + return plot_range + + bound_lower, bound_upper = expansion_bounds + if not np.isfinite(bound_lower) or not np.isfinite(bound_upper) or bound_lower >= bound_upper: + return plot_range + + lower = max(lower, bound_lower) + upper = min(upper, bound_upper) + if lower >= upper: + return plot_range + + if bins is None: + bins = int(np.clip(np.sqrt(finite_values.size), 10, 80)) + bins = max(1, int(bins)) + epsilon = max((bound_upper - bound_lower) * 1e-12, np.finfo(float).eps) + + for _ in range(max(0, int(max_steps)) + 1): + lower_fraction, upper_fraction, _ = self._histogram_edge_peak_fractions( + sample_values, + [lower, upper], + bins, + weights=weights, + ) + if not np.isfinite(lower_fraction) or not np.isfinite(upper_fraction): + return [float(lower), float(upper)] + + needs_lower = lower_fraction >= max_edge_peak_fraction + needs_upper = upper_fraction >= max_edge_peak_fraction + if not needs_lower and not needs_upper: + return [float(lower), float(upper)] + + width = upper - lower + if not np.isfinite(width) or width <= 0: + return [float(lower), float(upper)] + + new_lower = lower + new_upper = upper + if needs_lower and lower > bound_lower + epsilon: + new_lower = max(bound_lower, lower - width) + if needs_upper and upper < bound_upper - epsilon: + new_upper = min(bound_upper, upper + width) + + if new_lower == lower and new_upper == upper: + return [float(lower), float(upper)] + + lower, upper = new_lower, new_upper + + return [float(lower), float(upper)] + + def _get_mirrored_geometry_sample_cloud_range(self, sample_values, center, percentile_padding=0.5): + sample_values = np.asarray(sample_values, dtype=float) + finite_values = sample_values[np.isfinite(sample_values)] + if finite_values.size < 2: + return None + + try: + center = float(center) + except (TypeError, ValueError): + center = np.nan + if not np.isfinite(center): + return None + + percentile_padding = float(percentile_padding) + percentile_padding = min(max(percentile_padding, 0.0), 49.0) + q_lower, q_upper = np.nanpercentile( + finite_values, + [percentile_padding, 100.0 - percentile_padding], + ) + plot_lower = min(float(q_lower), center) + plot_upper = max(float(q_upper), center) + width = plot_upper - plot_lower + if not np.isfinite(width) or width <= 0: + return None + + padding = 0.05 * width + plot_lower -= padding + plot_upper += padding + max_distance = float(np.nanmax(np.abs([plot_lower - center, plot_upper - center]))) + if not np.isfinite(max_distance) or max_distance <= 0: + return None + return [-max_distance, max_distance] + + def _get_mirrored_geometry_full_range( + self, + key, + sample_values, + center, + sample_weights=None, + ): + try: + center = float(center) + except (TypeError, ValueError): + center = np.nan + if not np.isfinite(center): + return None + + plot_lower, plot_upper = self._get_plot_range(key) + plot_lower, plot_upper = self._expand_plot_range_for_sample_cloud( + key, + [plot_lower, plot_upper], + sample_values, + center, + required_visible_fraction=1.0, + ) + plot_bins = int(max(1, np.sqrt(np.asarray(sample_values).size))) + plot_lower, plot_upper = self._expand_plot_range_for_histogram_edge_dropoff( + key, + [plot_lower, plot_upper], + sample_values, + center, + bins=plot_bins, + weights=sample_weights, + ) + + max_distance = float(np.nanmax(np.abs([plot_lower - center, plot_upper - center]))) + if not np.isfinite(max_distance) or max_distance <= 0: + finite_offsets = np.asarray(sample_values, dtype=float) - center + finite_offsets = finite_offsets[np.isfinite(finite_offsets)] + if finite_offsets.size > 0: + max_distance = float(np.nanmax(np.abs(finite_offsets))) + if not np.isfinite(max_distance) or max_distance <= 0: + max_distance = max(abs(center) * 1e-6, 1e-6) + return [-max_distance, max_distance] + + def _get_triangle_plot_samples(self): + if self.ns_type == 'ultranest': + weighted_samples = self.results['weighted_samples'] + points = np.asarray(weighted_samples['points'], dtype=float) + logl = np.asarray(weighted_samples['logl'], dtype=float) + sampled_key_count = len(getattr(self, 'sampled_keys', [])) + if sampled_key_count and points.shape[1] >= sampled_key_count: + points = points[:, :sampled_key_count] + weights = self._get_triangle_plot_sample_weights( + weighted_samples.get('weights'), + points.shape[0], + ) + return points, logl, weights + + raise RuntimeError("Triangle plots require an UltraNest nested-sampling result.") + + def _get_triangle_plot_sample_weights(self, weights, sample_count): + if weights is None: + return None + + weights = np.asarray(weights, dtype=float) + if weights.ndim != 1 or weights.shape[0] != sample_count: + return None + + finite = np.isfinite(weights) & (weights >= 0) + if not np.all(finite): + weights = np.where(finite, weights, 0.0) + + if not np.isfinite(np.sum(weights)) or np.sum(weights) <= 0: + return None + return weights + + def get_parameter_posterior_samples(self, key): + sample_points = None + try: + equal_weight_samples = np.asarray(self.results.get('samples'), dtype=float) + if equal_weight_samples.ndim == 2 and equal_weight_samples.shape[0] > 0: + sample_points = equal_weight_samples + except Exception: + sample_points = None + if sample_points is None: + try: + sample_points, _, _ = self._get_triangle_plot_samples() + except Exception: + return np.array([], dtype=float) + + sample_points = np.asarray(sample_points, dtype=float) + if sample_points.ndim != 2 or sample_points.shape[0] == 0: + return np.array([], dtype=float) + + sampled_keys = list(getattr(self, 'sampled_keys', self._get_sampled_keys())) + if key in sampled_keys: + key_index = sampled_keys.index(key) + if key_index < sample_points.shape[1]: + return np.asarray(sample_points[:, key_index], dtype=float) + + bound_keys = list(self.bounds.keys()) + physical_samples = [ + self._physical_values_from_sample_point(point, bound_keys, sampled_keys).get(key, np.nan) + for point in sample_points + ] + return np.asarray(physical_samples, dtype=float) + + def _estimate_histogram_mode(self, samples, bounds=None, bins=None, weights=None): + samples = np.asarray(samples, dtype=float) + if weights is None: + finite_mask = np.isfinite(samples) + finite_weights = None + else: + weights = np.asarray(weights, dtype=float) + if weights.shape != samples.shape: + finite_mask = np.isfinite(samples) + finite_weights = None + else: + finite_mask = np.isfinite(samples) & np.isfinite(weights) & (weights >= 0) + finite_weights = weights[finite_mask] + if finite_weights.size == 0 or np.sum(finite_weights) <= 0: + finite_weights = None + + finite_samples = samples[finite_mask] + if finite_samples.size == 0: + return np.nan, np.nan + if finite_samples.size == 1: + return float(finite_samples[0]), np.nan + + if bounds is None: + lower = float(np.nanmin(finite_samples)) + upper = float(np.nanmax(finite_samples)) + else: + lower, upper = np.asarray(bounds, dtype=float).reshape(-1)[:2] + if not np.isfinite(lower) or not np.isfinite(upper) or lower >= upper: + lower = float(np.nanmin(finite_samples)) + upper = float(np.nanmax(finite_samples)) + + if not np.isfinite(lower) or not np.isfinite(upper) or lower >= upper: + return float(np.nanmedian(finite_samples)), np.nan + + if bins is None: + bins = int(np.clip(np.sqrt(finite_samples.size), 10, 80)) + bins = max(1, int(bins)) + + counts, edges = np.histogram(finite_samples, bins=bins, range=(lower, upper), weights=finite_weights) + if counts.size == 0: + return float(np.nanmedian(finite_samples)), np.nan + if not np.any(counts > 0): + return float(np.nanmedian(finite_samples)), np.nan + + mode_index = int(np.argmax(counts)) + mode = float(0.5 * (edges[mode_index] + edges[mode_index + 1])) + bin_width = float(edges[1] - edges[0]) if edges.size > 1 else np.nan + return mode, bin_width + + def _format_triangle_plot_parameter_title(self, value, error): + try: + value = float(value) + except (TypeError, ValueError): + return "n/a" + try: + error = float(error) + except (TypeError, ValueError): + error = np.nan + if not np.isfinite(value): + return "n/a" + if not np.isfinite(error) or error < 0: + return str(round_to_2(value)) + return format_value_with_uncertainty(value, error) + + def _weighted_quantiles(self, values, quantiles, weights=None): + values = np.asarray(values, dtype=float) + quantiles = np.asarray(quantiles, dtype=float) + finite_mask = np.isfinite(values) + + finite_weights = None + if weights is not None: + weights = np.asarray(weights, dtype=float) + if weights.shape == values.shape: + finite_mask &= np.isfinite(weights) & (weights >= 0) + finite_weights = weights[finite_mask] + if finite_weights.size == 0 or np.sum(finite_weights) <= 0: + finite_weights = None + + finite_values = values[finite_mask] + if finite_values.size == 0: + return np.full(quantiles.shape, np.nan, dtype=float) + if finite_weights is None: + return np.nanpercentile(finite_values, 100.0 * quantiles) + + order = np.argsort(finite_values) + sorted_values = finite_values[order] + sorted_weights = finite_weights[order] + cumulative = np.cumsum(sorted_weights) + total = cumulative[-1] + if not np.isfinite(total) or total <= 0: + return np.nanpercentile(finite_values, 100.0 * quantiles) + + cumulative = (cumulative - 0.5 * sorted_weights) / total + cumulative = np.clip(cumulative, 0.0, 1.0) + return np.interp(quantiles, cumulative, sorted_values) + + def _triangle_plot_display_estimate( + self, + sample_values, + fallback_center, + fallback_error, + plot_range=None, + weights=None, + min_informative_peak_ratio=1.5, + bins=None, + force_histogram_mode=False, + ): + sample_values = np.asarray(sample_values, dtype=float) + finite_mask = np.isfinite(sample_values) + finite_weights = None + if weights is not None: + weights = np.asarray(weights, dtype=float) + if weights.shape == sample_values.shape: + finite_mask &= np.isfinite(weights) & (weights >= 0) + finite_weights = weights[finite_mask] + if finite_weights.size == 0 or np.sum(finite_weights) <= 0: + finite_weights = None + + finite_values = sample_values[finite_mask] + if finite_values.size < 2: + return fallback_center, fallback_error + + q16, q50, q84 = self._weighted_quantiles( + sample_values, + [0.158655, 0.5, 0.841345], + weights=weights, + ) + estimate = q50 + if plot_range is None: + bounds = [float(np.nanmin(finite_values)), float(np.nanmax(finite_values))] + else: + try: + bounds = [float(value) for value in np.asarray(plot_range, dtype=float).reshape(-1)[:2]] + except (TypeError, ValueError, IndexError): + bounds = [float(np.nanmin(finite_values)), float(np.nanmax(finite_values))] + + if np.all(np.isfinite(bounds)) and bounds[0] < bounds[1]: + if bins is None: + bins = int(np.clip(np.sqrt(finite_values.size), 10, 80)) + else: + bins = max(1, int(bins)) + counts, edges = np.histogram( + finite_values, + bins=max(1, bins), + range=bounds, + weights=finite_weights, + ) + positive_counts = counts[counts > 0] + if positive_counts.size > 0: + peak = float(np.nanmax(positive_counts)) + typical = float(np.nanmedian(positive_counts)) + total = float(np.nansum(positive_counts)) + informative_peak = ( + np.isfinite(peak) + and np.isfinite(typical) + and np.isfinite(total) + and total > 0 + and ( + force_histogram_mode + or ( + typical > 0 + and peak >= min_informative_peak_ratio * typical + and peak >= 0.05 * total + ) + ) + ) + if informative_peak: + mode_index = int(np.argmax(counts)) + estimate = float(0.5 * (edges[mode_index] + edges[mode_index + 1])) + + try: + fallback_center = float(fallback_center) + except (TypeError, ValueError): + fallback_center = np.nan + if not np.isfinite(estimate): + estimate = fallback_center + + spread_candidates = [ + abs(float(q84) - float(estimate)) if np.isfinite(q84) and np.isfinite(estimate) else np.nan, + abs(float(estimate) - float(q16)) if np.isfinite(q16) and np.isfinite(estimate) else np.nan, + 0.5 * abs(float(q84) - float(q16)) if np.isfinite(q16) and np.isfinite(q84) else np.nan, + ] + try: + fallback_error = float(fallback_error) + except (TypeError, ValueError): + fallback_error = np.nan + + finite_spreads = [value for value in spread_candidates if np.isfinite(value) and value >= 0] + if finite_spreads and max(finite_spreads) > 0: + error = float(max(finite_spreads)) + elif np.isfinite(fallback_error) and fallback_error > 0: + error = fallback_error + else: + error = np.nan + + return float(estimate), error + + def _visible_triangle_plot_values(self, values, plot_range, weights=None): + values = np.asarray(values, dtype=float) + try: + lower, upper = [float(value) for value in np.asarray(plot_range, dtype=float).reshape(-1)[:2]] + except (TypeError, ValueError, IndexError): + finite_mask = np.isfinite(values) + return values[finite_mask], None + + finite_mask = np.isfinite(values) + if np.isfinite(lower) and np.isfinite(upper) and lower < upper: + finite_mask &= (values >= lower) & (values <= upper) + + visible_weights = None + if weights is not None: + weights = np.asarray(weights, dtype=float) + if weights.shape == values.shape: + visible_weights = weights[finite_mask] + return values[finite_mask], visible_weights + + def get_parameter_posterior_recenter_diagnostics(self, key, sigma_scale=5.0, bins=None): + diagnostics = { + 'key': key, + 'clipped': False, + 'edge': None, + 'mode': np.nan, + 'std': np.nan, + 'full_std': np.nan, + 'bounds': None, + 'original_bounds': None, + 'sample_size': 0, + 'peak_height': np.nan, + 'lower_edge_height': np.nan, + 'upper_edge_height': np.nan, + 'lower_edge_peak_fraction': np.nan, + 'upper_edge_peak_fraction': np.nan, + 'reason': None, + } + + bounds = getattr(self, 'sample_bounds', {}).get(key, self.bounds.get(key)) + if bounds is None: + diagnostics['reason'] = "parameter bounds are unavailable." + return diagnostics + + try: + lower_bound, upper_bound = np.asarray(bounds, dtype=float).reshape(-1)[:2] + except (TypeError, ValueError, IndexError): + diagnostics['reason'] = "parameter bounds are malformed." + return diagnostics + + diagnostics['original_bounds'] = [float(lower_bound), float(upper_bound)] + if not np.isfinite(lower_bound) or not np.isfinite(upper_bound) or lower_bound >= upper_bound: + diagnostics['reason'] = "parameter bounds are not finite." + return diagnostics + + samples = self.get_parameter_posterior_samples(key) + finite_samples = np.asarray(samples, dtype=float) + finite_samples = finite_samples[np.isfinite(finite_samples)] + diagnostics['sample_size'] = int(finite_samples.size) + if finite_samples.size < 8: + diagnostics['reason'] = "too few posterior samples are available." + return diagnostics + + mode, bin_width = self._estimate_histogram_mode(finite_samples, bounds=(lower_bound, upper_bound), bins=bins) + full_std = float(np.nanstd(finite_samples)) + diagnostics['mode'] = mode + diagnostics['full_std'] = full_std + + if not np.isfinite(mode): + diagnostics['reason'] = "posterior mode could not be estimated." + return diagnostics + + q05, q16, q50, q84, q95 = np.nanpercentile(finite_samples, [5, 16, 50, 84, 95]) + width = float(upper_bound - lower_bound) + histogram_bins = int(np.clip(np.sqrt(finite_samples.size), 10, 80)) if bins is None else max(1, int(bins)) + histogram_counts, _ = np.histogram( + finite_samples, + bins=histogram_bins, + range=(lower_bound, upper_bound), + ) + histogram_counts = np.asarray(histogram_counts, dtype=float) + peak_height = float(np.nanmax(histogram_counts)) if histogram_counts.size else np.nan + lower_edge_height = float(histogram_counts[0]) if histogram_counts.size else np.nan + upper_edge_height = float(histogram_counts[-1]) if histogram_counts.size else np.nan + if np.isfinite(peak_height) and peak_height > 0: + lower_edge_peak_fraction = float(lower_edge_height / peak_height) + upper_edge_peak_fraction = float(upper_edge_height / peak_height) + else: + lower_edge_peak_fraction = np.nan + upper_edge_peak_fraction = np.nan + + diagnostics['peak_height'] = peak_height + diagnostics['lower_edge_height'] = lower_edge_height + diagnostics['upper_edge_height'] = upper_edge_height + diagnostics['lower_edge_peak_fraction'] = lower_edge_peak_fraction + diagnostics['upper_edge_peak_fraction'] = upper_edge_peak_fraction + + scale_floor = max( + 2.0 * bin_width if np.isfinite(bin_width) and bin_width > 0 else 0.0, + 0.01 * width, + np.finfo(float).eps, + ) + tail_gap_threshold = max(0.5 * full_std if np.isfinite(full_std) and full_std > 0 else 0.0, scale_floor) + mode_gap_threshold = max( + 1.0 * full_std if np.isfinite(full_std) and full_std > 0 else 0.0, + 3.0 * bin_width if np.isfinite(bin_width) and bin_width > 0 else 0.0, + 0.05 * width, + np.finfo(float).eps, + ) + + upper_gap_q95 = float(upper_bound - q95) + lower_gap_q05 = float(q05 - lower_bound) + upper_gap_mode = float(upper_bound - mode) + lower_gap_mode = float(mode - lower_bound) + + upper_clipped = upper_gap_q95 <= tail_gap_threshold and upper_gap_mode <= mode_gap_threshold + lower_clipped = lower_gap_q05 <= tail_gap_threshold and lower_gap_mode <= mode_gap_threshold + edge_peak_fraction_floor = 0.20 + rejected_edges = [] + + if upper_clipped and np.isfinite(upper_edge_peak_fraction) and upper_edge_peak_fraction < edge_peak_fraction_floor: + upper_clipped = False + rejected_edges.append( + f"upper edge histogram height is only {upper_edge_peak_fraction:.3f} of the posterior peak" + ) + if lower_clipped and np.isfinite(lower_edge_peak_fraction) and lower_edge_peak_fraction < edge_peak_fraction_floor: + lower_clipped = False + rejected_edges.append( + f"lower edge histogram height is only {lower_edge_peak_fraction:.3f} of the posterior peak" + ) + + if upper_clipped and lower_clipped: + clipped_edge = 'upper' if upper_gap_mode <= lower_gap_mode else 'lower' + elif upper_clipped: + clipped_edge = 'upper' + elif lower_clipped: + clipped_edge = 'lower' + else: + diagnostics['bounds'] = [float(lower_bound), float(upper_bound)] + if rejected_edges: + diagnostics['reason'] = ( + "posterior reaches a search bound, but " + + " and ".join(rejected_edges) + + ", so it is not treated as truncated." + ) + else: + diagnostics['reason'] = "posterior support is comfortably inside the sampled bounds." + return diagnostics + + diagnostics['clipped'] = True + diagnostics['edge'] = clipped_edge + + if clipped_edge == 'upper': + side_distances = mode - finite_samples[finite_samples <= mode] + else: + side_distances = finite_samples[finite_samples >= mode] - mode + + side_distances = np.asarray(side_distances, dtype=float) + side_distances = side_distances[np.isfinite(side_distances)] + side_distances = side_distances[side_distances >= 0] + + if side_distances.size >= 2: + mirrored = np.concatenate([side_distances, -side_distances]) + estimated_std = float(np.nanstd(mirrored)) + else: + estimated_std = full_std + + min_std = max( + bin_width if np.isfinite(bin_width) and bin_width > 0 else 0.0, + width * 1e-3, + np.finfo(float).eps, + ) + if not np.isfinite(estimated_std) or estimated_std <= 0: + estimated_std = full_std + if not np.isfinite(estimated_std) or estimated_std <= 0: + estimated_std = min_std + estimated_std = float(max(estimated_std, min_std)) + diagnostics['std'] = estimated_std + + radius = float(max(sigma_scale * estimated_std, min_std)) + new_lower = float(mode - radius) + new_upper = float(mode + radius) + if lower_bound >= 0: + new_lower = max(0.0, new_lower) + diagnostics['bounds'] = [new_lower, new_upper] + diagnostics['reason'] = ( + f"posterior peaks against the {clipped_edge} search bound " + f"(mode={mode:.6g}, sigma={estimated_std:.6g})." + ) + diagnostics['q16'] = float(q16) + diagnostics['q50'] = float(q50) + diagnostics['q84'] = float(q84) + diagnostics['q05'] = float(q05) + diagnostics['q95'] = float(q95) + return diagnostics + + def _get_triangle_plot_display_spec( + self, + sampled_keys, + sample_parameters, + sample_errors, + sample_points, + sample_weights=None, + ): + if 'b' in sampled_keys: + key = 'b' + label = r'Impact parameter $b$' + mirror = False + elif 'inc' in sampled_keys: + key = 'inc' + label = r'$\Delta i$' + mirror = True + else: + return None + + geometry_index = sampled_keys.index(key) + center = float(sample_parameters.get(key, self.parameters.get(key, 0.0))) + sample_values = np.asarray(sample_points[:, geometry_index], dtype=float) + magnitude_samples = np.abs(sample_values - center) + error = float(sample_errors.get(key, np.nanstd(magnitude_samples))) + if not np.isfinite(error) or error <= 0: + error = float(np.nanstd(magnitude_samples)) + + if mirror: + display_range = self._get_mirrored_geometry_full_range( + key, + sample_values, + center, + sample_weights=sample_weights, + ) + else: + display_range = None + + if display_range is None: + plot_lower, plot_upper = self._get_plot_range(key) + plot_lower, plot_upper = self._expand_plot_range_for_sample_cloud( + key, + [plot_lower, plot_upper], + sample_values, + center, + ) + plot_bins = int(max(1, np.sqrt(sample_points.shape[0]))) + plot_lower, plot_upper = self._expand_plot_range_for_histogram_edge_dropoff( + key, + [plot_lower, plot_upper], + sample_values, + center, + bins=plot_bins, + weights=sample_weights, + ) + if mirror: + max_distance = float(np.nanmax(np.abs([plot_lower - center, plot_upper - center]))) + if not np.isfinite(max_distance) or max_distance <= 0: + max_distance = float(np.nanmax(magnitude_samples)) + if not np.isfinite(max_distance) or max_distance <= 0: + max_distance = max(abs(center) * 1e-6, 1e-6) + display_range = [-max_distance, max_distance] + else: + display_range = [float(plot_lower), float(plot_upper)] + return { + 'key': key, + 'index': geometry_index, + 'label': label, + 'mirror': mirror, + 'center': center, + 'mask_center': 0.0 if mirror else center, + 'mask_error': error, + 'magnitude_samples': magnitude_samples, + 'range': display_range, + 'truth': 0.0 if mirror else center, + 'reference_lines': self._get_triangle_plot_geometry_reference_lines( + key, + center, + sample_parameters, + ), + } + + def _get_triangle_plot_geometry_reference_lines(self, key, center, sample_parameters): + if key != 'b': + return [] + + try: + center = float(center) + except (TypeError, ValueError): + center = np.nan + if not np.isfinite(center): + return [] + + rprs = sample_parameters.get('rprs') + if rprs is None: + rprs = getattr(self, 'parameters', {}).get( + 'rprs', + getattr(self, 'prior', {}).get('rprs', np.nan), + ) + try: + rprs = float(rprs) + except (TypeError, ValueError): + rprs = np.nan + + reference_lines = [ + { + 'value': 1.0, + 'color': '#707070', + 'linestyle': ':', + 'linewidth': 0.9, + 'alpha': 0.9, + }, + ] + if np.isfinite(rprs) and rprs >= 0: + reference_lines.append( + { + 'value': 1.0 + rprs, + 'color': '#a35d00', + 'linestyle': '-.', + 'linewidth': 0.9, + 'alpha': 0.9, + } + ) + return reference_lines + + def _get_triangle_plot_geometry_reference_offsets(self, display_spec): + reference_lines = display_spec.get('reference_lines', []) if isinstance(display_spec, dict) else [] + if not reference_lines: + return [] + + try: + center = float(display_spec['center']) + except (KeyError, TypeError, ValueError): + return [] + if not np.isfinite(center): + return [] + + offsets = [] + for reference in reference_lines: + try: + value = float(reference['value']) + except (KeyError, TypeError, ValueError): + continue + if not np.isfinite(value): + continue + + distance = abs(value - center) + if not np.isfinite(distance): + continue + reference_offsets = [0.0] if distance <= np.finfo(float).eps else [-distance, distance] + for offset in reference_offsets: + offsets.append({ + 'offset': float(offset), + 'color': reference.get('color', '#707070'), + 'linestyle': reference.get('linestyle', ':'), + 'linewidth': reference.get('linewidth', 0.9), + 'alpha': reference.get('alpha', 0.9), + }) + return offsets + + def _draw_triangle_plot_geometry_reference_lines(self, ax, display_spec, axis='x', limits=None): + if ax is None: + return + + if limits is None: + limits = ax.get_xlim() if axis == 'x' else ax.get_ylim() + lower, upper = np.sort(np.asarray(limits, dtype=float).reshape(-1)[:2]) + if not display_spec.get('mirror', True): + for reference in display_spec.get('reference_lines', []): + try: + value = float(reference['value']) + except (KeyError, TypeError, ValueError): + continue + if not np.isfinite(value) or value < lower or value > upper: + continue + line_kwargs = { + 'color': reference.get('color', '#707070'), + 'linestyle': reference.get('linestyle', ':'), + 'linewidth': reference.get('linewidth', 0.9), + 'alpha': reference.get('alpha', 0.9), + 'zorder': 2, + } + if axis == 'y': + ax.axhline(value, **line_kwargs) + else: + ax.axvline(value, **line_kwargs) + return + + if lower <= 0.0 <= upper: + center_kwargs = { + 'color': '#4682b4', + 'linestyle': '--', + 'linewidth': 0.9, + 'alpha': 0.85, + 'zorder': 2, + } + if axis == 'y': + ax.axhline(0.0, **center_kwargs) + else: + ax.axvline(0.0, **center_kwargs) + for reference in self._get_triangle_plot_geometry_reference_offsets(display_spec): + offset = reference['offset'] + if offset < lower or offset > upper: + continue + line_kwargs = { + 'color': reference['color'], + 'linestyle': reference['linestyle'], + 'linewidth': reference['linewidth'], + 'alpha': reference['alpha'], + 'zorder': 2, + } + if axis == 'y': + ax.axhline(offset, **line_kwargs) + else: + ax.axvline(offset, **line_kwargs) + + def _get_triangle_plot_geometry_overlay(self, display_spec, sample_points): + if display_spec is None: + return None + if not display_spec.get('mirror', True): + return None + + geometry_index = display_spec['index'] + center = display_spec['center'] + offsets = np.asarray(sample_points[:, geometry_index], dtype=float) - center + left_offsets = offsets[offsets <= 0] + right_offsets = offsets[offsets >= 0] + + def mirrored_offsets(branch_offsets): + if branch_offsets.size == 0: + return np.array([], dtype=float) + return np.concatenate([branch_offsets, -branch_offsets]) + + return { + 'index': geometry_index, + 'left_count': left_offsets.size, + 'right_count': right_offsets.size, + 'left_mirrored': mirrored_offsets(left_offsets), + 'right_mirrored': mirrored_offsets(right_offsets), + } + + def _format_triangle_plot_geometry_value(self, value, error, suffix=''): + if value is None or not np.isfinite(value): + return f"n/a{suffix}" + if error is None or not np.isfinite(error) or error < 0: + return f"{round_to_2(value)}{suffix}" + return f"{format_value_with_uncertainty(value, error)}{suffix}" + + def _get_triangle_plot_geometry_summary(self, sampled_keys, sample_points): + bound_keys = list(self.bounds.keys()) + sample_points = np.asarray(sample_points, dtype=float) + sample_parameters = getattr(self, 'sample_parameters', {}) + sample_errors = getattr(self, 'sample_errors', {}) + + physical_samples = [ + self._physical_values_from_sample_point(point, bound_keys, sampled_keys) + for point in sample_points + ] + inc_samples = np.array([sample['inc'] for sample in physical_samples], dtype=float) + b_samples = np.array([ + sample.get('b', impact_parameter_from_inclination(sample, sample['inc'])) + for sample in physical_samples + ], dtype=float) + + inc_center = float(self.parameters.get('inc', np.nanmedian(inc_samples))) + inc_error = float(self.errors.get('inc', np.nanstd(inc_samples))) + if 'b' in sample_parameters: + b_center = float(sample_parameters['b']) + elif 'b' in self.parameters: + b_center = float(self.parameters['b']) + else: + b_center = float(np.nanmedian(b_samples)) + b_error = float(sample_errors.get('b', self.errors.get('b', np.nanstd(b_samples)))) + + title = ( + f"b={self._format_triangle_plot_geometry_value(b_center, b_error)}\n" + f"i={self._format_triangle_plot_geometry_value(inc_center, inc_error, ' deg')}" + ) + return { + 'b_center': b_center, + 'b_error': b_error, + 'inc_center': inc_center, + 'inc_error': inc_error, + 'title': title, + } + + def _smooth_triangle_plot_counts(self, counts): + counts = np.asarray(counts, dtype=float) + if counts.size <= 1 or not np.any(counts > 0): + return counts + + sigma_bins = max(1.0, counts.size / 18.0) + radius = max(1, int(np.ceil(3 * sigma_bins))) + grid = np.arange(-radius, radius + 1, dtype=float) + kernel = np.exp(-0.5 * (grid / sigma_bins) ** 2) + kernel /= np.sum(kernel) + return np.convolve(counts, kernel, mode='same') + + def _build_triangle_plot_geometry_curves(self, geometry_overlay, hist_range, bins_1d): + hist_range = np.sort(np.asarray(hist_range, dtype=float)) + bins_1d = max(1, int(bins_1d)) + edges = np.linspace(hist_range[0], hist_range[1], bins_1d + 1) + centers = 0.5 * (edges[:-1] + edges[1:]) + half_bin = 0.5 * (edges[1] - edges[0]) if edges.size > 1 else 0.0 + + def branch_curve(samples): + samples = np.asarray(samples, dtype=float) + counts, _ = np.histogram(samples, bins=edges) + support = np.zeros_like(counts, dtype=bool) + if samples.size > 0: + support = np.abs(centers) <= (np.max(np.abs(samples)) + half_bin) + return counts.astype(float), support + + left_curve, left_support = branch_curve(geometry_overlay['left_mirrored']) + right_curve, right_support = branch_curve(geometry_overlay['right_mirrored']) + + left_count = int(geometry_overlay.get('left_count', 0)) + right_count = int(geometry_overlay.get('right_count', 0)) + max_count = max(left_count, right_count) + min_count = min(left_count, right_count) + + if max_count == 0: + main_curve = np.zeros_like(centers, dtype=float) + elif min_count == 0 or (min_count / max_count) < 0.35: + main_curve = left_curve if left_count >= right_count else right_curve + else: + stacked = np.vstack([ + np.where(left_support, left_curve, np.nan), + np.where(right_support, right_curve, np.nan), + ]) + valid_counts = np.sum(np.isfinite(stacked), axis=0) + summed = np.nansum(stacked, axis=0) + main_curve = np.divide( + summed, + valid_counts, + out=np.zeros_like(summed, dtype=float), + where=valid_counts > 0, + ) + + main_curve = self._smooth_triangle_plot_counts(main_curve) + + return { + 'centers': centers, + 'left_curve': left_curve, + 'right_curve': right_curve, + 'main_curve': main_curve, + } + + def _get_triangle_plot_payload(self): + sampled_keys = getattr(self, 'sampled_keys', list(self.bounds.keys())) + sample_parameters = getattr(self, 'sample_parameters', self.parameters) + sample_errors = getattr(self, 'sample_errors', self.errors) + sample_points, sample_logl, sample_weights = self._get_triangle_plot_samples() + display_spec = self._get_triangle_plot_display_spec( + sampled_keys, + sample_parameters, + sample_errors, + sample_points, + sample_weights=sample_weights, + ) + geometry_overlay = self._get_triangle_plot_geometry_overlay(display_spec, sample_points) + geometry_summary = self._get_triangle_plot_geometry_summary(sampled_keys, sample_points) + + display_points = np.array(sample_points, copy=True) + display_logl = np.array(sample_logl, copy=True) + display_weights = None if sample_weights is None else np.array(sample_weights, copy=True) + mask_values = np.array(sample_points, copy=True) + + if display_spec is not None and display_spec.get('mirror', True): + geometry_index = display_spec['index'] + positive_points = np.array(sample_points, copy=True) + negative_points = np.array(sample_points, copy=True) + positive_points[:, geometry_index] = display_spec['magnitude_samples'] + negative_points[:, geometry_index] = -display_spec['magnitude_samples'] + display_points = np.vstack([positive_points, negative_points]) + display_logl = np.concatenate([sample_logl, sample_logl]) + if sample_weights is not None: + display_weights = np.concatenate([sample_weights, sample_weights]) + mask_values = np.array(display_points, copy=True) + + plot_bins = int(max(1, np.sqrt(display_points.shape[0]))) + + flabels = { + 'rprs': r'R$_{p}$/R$_{s}$', + 'per': r'Period [day]', + 'tmid': r'T$_{mid}$', + 'ars': r'a/R$_{s}$', + 'inc': r'Inc. [deg]', + 'b': r'Impact parameter', + 'u1': r'u$_1$', + 'fpfs': r'F$_{p}$/F$_{s}$', + 'omega': r'$\omega$ [deg]', + 'mplanet': r'M$_{p}$ [M$_{\oplus}$]', + 'mstar': r'M$_{s}$ [M$_{\odot}$]', + 'ecc': r'$e$', + 'c0': r'$c_0$', + 'c1': r'$c_1$', + 'c2': r'$c_2$', + 'c3': r'$c_3$', + 'c4': r'$c_4$', + 'a0': r'$a_0$', + 'a1': r'$a_1$', + 'a2': r'$a_2$' + } + + labels = [] + titles = [] + ranges = [] + mask_centers = [] + mask_errors = [] + truths = [] + + for i, key in enumerate(sampled_keys): + center = sample_parameters.get(key, self.parameters.get(key, 0.0)) + error = sample_errors.get(key, self.errors.get(key, 0.0)) + label = flabels.get(key, key) + plot_range = self._get_plot_range(key) + if sample_points.ndim == 2 and i < sample_points.shape[1]: + plot_range = self._expand_plot_range_for_sample_cloud( + key, + plot_range, + sample_points[:, i], + center, + ) + plot_range = self._expand_plot_range_for_histogram_edge_dropoff( + key, + plot_range, + sample_points[:, i], + center, + bins=plot_bins, + weights=sample_weights, + ) + center, error = self._triangle_plot_display_estimate( + sample_points[:, i], + center, + error, + plot_range=plot_range, + weights=sample_weights, + ) + title = self._format_triangle_plot_parameter_title(center, error) + truth = center + + if display_spec is not None and key == display_spec['key']: + label = display_spec['label'] + title = geometry_summary['title'] + plot_range = display_spec['range'] + center = display_spec['mask_center'] + error = display_spec['mask_error'] + truth = display_spec['truth'] + + labels.append(label) + titles.append(title) + ranges.append(plot_range) + mask_centers.append(center) + mask_errors.append(error) + try: + truth = float(truth) + except (TypeError, ValueError): + truth = np.nan + truths.append(truth if np.isfinite(truth) else None) + + return { + 'sampled_keys': sampled_keys, + 'display_points': display_points, + 'display_logl': display_logl, + 'display_weights': display_weights, + 'mask_values': mask_values, + 'display_spec': display_spec, + 'geometry_overlay': geometry_overlay, + 'geometry_summary': geometry_summary, + 'labels': labels, + 'titles': titles, + 'ranges': ranges, + 'mask_centers': mask_centers, + 'mask_errors': mask_errors, + 'truths': truths, + } + + def _triangle_plot_sigma_window_ranges(self, payload, sigma): + try: + sigma = float(sigma) + except (TypeError, ValueError): + return payload['ranges'] + if not np.isfinite(sigma) or sigma <= 0: + return payload['ranges'] + + zoomed_ranges = [] + display_points = np.asarray(payload.get('display_points', []), dtype=float) + for i, plot_range in enumerate(payload['ranges']): + try: + range_lower, range_upper = [ + float(value) for value in np.asarray(plot_range, dtype=float).reshape(-1)[:2] + ] + except (TypeError, ValueError, IndexError): + zoomed_ranges.append(plot_range) + continue + + if not np.isfinite(range_lower) or not np.isfinite(range_upper) or range_lower >= range_upper: + zoomed_ranges.append(plot_range) + continue + + try: + center = float(payload['mask_centers'][i]) + error = float(payload['mask_errors'][i]) + except (TypeError, ValueError, IndexError): + zoomed_ranges.append(plot_range) + continue + + if not np.isfinite(center) or not np.isfinite(error) or error <= 0: + zoomed_ranges.append(plot_range) + continue + + lower = max(range_lower, center - sigma * error) + upper = min(range_upper, center + sigma * error) + if not np.isfinite(lower) or not np.isfinite(upper) or lower >= upper: + zoomed_ranges.append(plot_range) + continue + + if display_points.ndim == 2 and i < display_points.shape[1]: + values = display_points[:, i] + finite_values = values[np.isfinite(values)] + if finite_values.size and not np.any((finite_values >= lower) & (finite_values <= upper)): + zoomed_ranges.append(plot_range) + continue + + zoomed_ranges.append([float(lower), float(upper)]) + + return zoomed_ranges + + def _recenter_triangle_plot_payload_for_visible_ranges(self, payload): + display_points = np.asarray(payload.get('display_points', []), dtype=float) + if display_points.ndim != 2 or display_points.shape[1] == 0: + return payload + + updated = dict(payload) + titles = list(payload.get('titles', [])) + truths = list(payload.get('truths', [])) + mask_centers = list(payload.get('mask_centers', [])) + mask_errors = list(payload.get('mask_errors', [])) + ranges = list(payload.get('ranges', [])) + sampled_keys = list(payload.get('sampled_keys', [])) + + display_weights = payload.get('display_weights') + if display_weights is not None: + display_weights = np.asarray(display_weights, dtype=float) + if display_weights.ndim != 1 or display_weights.shape[0] != display_points.shape[0]: + display_weights = None + + plot_bins = int(max(1, np.sqrt(display_points.shape[0]))) + display_spec = payload.get('display_spec') + geometry_summary = payload.get('geometry_summary') or {} + + for i, key in enumerate(sampled_keys): + if i >= display_points.shape[1] or i >= len(ranges): + continue + if ( + display_spec is not None + and key == display_spec.get('key') + and display_spec.get('mirror', False) + ): + continue + + visible_values, visible_weights = self._visible_triangle_plot_values( + display_points[:, i], + ranges[i], + weights=display_weights, + ) + if visible_values.size < 2: + continue + + fallback_center = truths[i] if i < len(truths) else np.nan + if fallback_center is None or not np.isfinite(fallback_center): + fallback_center = mask_centers[i] if i < len(mask_centers) else np.nan + fallback_error = mask_errors[i] if i < len(mask_errors) else np.nan + center, error = self._triangle_plot_display_estimate( + visible_values, + fallback_center, + fallback_error, + plot_range=ranges[i], + weights=visible_weights, + bins=plot_bins, + force_histogram_mode=True, + ) + if not np.isfinite(center): + continue + + if i < len(truths): + truths[i] = center + if i < len(mask_centers): + mask_centers[i] = center + if i < len(mask_errors): + mask_errors[i] = error + if i < len(titles): + if display_spec is not None and key == display_spec.get('key'): + inc_center = geometry_summary.get('inc_center') + inc_error = geometry_summary.get('inc_error') + titles[i] = ( + f"b={self._format_triangle_plot_geometry_value(center, error)}\n" + f"i={self._format_triangle_plot_geometry_value(inc_center, inc_error, ' deg')}" + ) + else: + titles[i] = self._format_triangle_plot_parameter_title(center, error) + + updated['titles'] = titles + updated['truths'] = truths + updated['mask_centers'] = mask_centers + updated['mask_errors'] = mask_errors + return updated + + def _triangle_contour_levels(self, chi2, mask1, mask2, mask3): + raw_levels = np.array([ + np.percentile(chi2[mask1], 95), + np.percentile(chi2[mask2], 95), + np.percentile(chi2[mask3], 95), + ], dtype=float) + finite_levels = np.sort(raw_levels[np.isfinite(raw_levels)]) + if finite_levels.size == 0: + return [] + + unique_levels = [] + min_spacing = max( + np.finfo(float).eps, + np.nanmax(np.abs(finite_levels)) * 1e-12, + ) + for level in finite_levels: + if not unique_levels or level > unique_levels[-1] + min_spacing: + unique_levels.append(float(level)) + return unique_levels + + def _overlay_triangle_plot_geometry_histograms(self, fig, payload, title_kwargs=None, label_kwargs=None): + if not hasattr(fig, 'axes'): + return + + display_spec = payload.get('display_spec') + if display_spec is None: + return + + sampled_keys = payload['sampled_keys'] + if len(fig.axes) != len(sampled_keys) ** 2: + return + + axes = np.array(fig.axes).reshape((len(sampled_keys), len(sampled_keys))) + if not display_spec.get('mirror', True): + geometry_index = display_spec['index'] + for row in range(len(sampled_keys)): + for col in range(len(sampled_keys)): + panel = axes[row, col] + if row == geometry_index and col == geometry_index: + self._draw_triangle_plot_geometry_reference_lines(panel, display_spec, axis='x') + elif col == geometry_index and row > col: + self._draw_triangle_plot_geometry_reference_lines(panel, display_spec, axis='x') + elif row == geometry_index and col < row: + self._draw_triangle_plot_geometry_reference_lines(panel, display_spec, axis='y') + return + + geometry_overlay = payload.get('geometry_overlay') + if geometry_overlay is None: + return + + geometry_index = geometry_overlay['index'] + ax = axes[geometry_index, geometry_index] + hist_range = np.sort(payload['ranges'][geometry_index]) + bins_1d = int(max(25, np.round(np.sqrt(payload['display_points'].shape[0]) * 6))) + curves = self._build_triangle_plot_geometry_curves(geometry_overlay, hist_range, bins_1d) + + title = payload['titles'][geometry_index] + x_label = payload['labels'][geometry_index] + branch_left_color = '#6f8fcf' + branch_right_color = '#d79b9b' + title_kwargs = {} if title_kwargs is None else dict(title_kwargs) + label_kwargs = {} if label_kwargs is None else dict(label_kwargs) + + ax.cla() + ax.plot(curves['centers'], curves['main_curve'], color='black', linewidth=1.5, zorder=4) + ax.plot(curves['centers'], curves['left_curve'], color=branch_left_color, linestyle='--', + linewidth=0.75, alpha=0.75, zorder=3) + ax.plot(curves['centers'], curves['right_curve'], color=branch_right_color, linestyle='--', + linewidth=0.75, alpha=0.75, zorder=3) + self._draw_triangle_plot_geometry_reference_lines( + ax, + display_spec, + axis='x', + limits=hist_range, + ) + ax.set_title(title, **title_kwargs) + if 'fontsize' in title_kwargs: + ax.title.set_fontsize(title_kwargs['fontsize']) + ax.set_xlim(hist_range) + + max_y = max( + np.max(curves['main_curve']) if curves['main_curve'].size > 0 else 0.0, + np.max(curves['left_curve']) if curves['left_curve'].size > 0 else 0.0, + np.max(curves['right_curve']) if curves['right_curve'].size > 0 else 0.0, + ) + ax.set_ylim(0, 1.1 * max(max_y, 1e-6)) + ax.set_yticks([]) + + if geometry_index < len(sampled_keys) - 1: + ax.set_xticklabels([]) + else: + ax.set_xlabel(x_label, **label_kwargs) + + for row in range(len(sampled_keys)): + for col in range(len(sampled_keys)): + if row == geometry_index and col == geometry_index: + continue + panel = axes[row, col] + if col == geometry_index and row > col: + self._draw_triangle_plot_geometry_reference_lines(panel, display_spec, axis='x') + if row == geometry_index and col < row: + self._draw_triangle_plot_geometry_reference_lines(panel, display_spec, axis='y') + + def _overlay_single_parameter_triangle_gaussian(self, fig, payload): + if not hasattr(fig, 'axes') or len(fig.axes) != 1: + return + sampled_keys = list(payload.get('sampled_keys', [])) + if len(sampled_keys) != 1: + return + + ax = fig.axes[0] + display_points = np.asarray(payload.get('display_points', []), dtype=float) + if display_points.ndim != 2 or display_points.shape[1] != 1: + return + + values = display_points[:, 0] + values = values[np.isfinite(values)] + if values.size < 2: + return + + try: + center = float(payload.get('mask_centers', [np.nan])[0]) + sigma = float(payload.get('mask_errors', [np.nan])[0]) + lower, upper = [ + float(value) + for value in np.asarray(payload.get('ranges', [[np.nan, np.nan]])[0], dtype=float).reshape(-1)[:2] + ] + except (TypeError, ValueError, IndexError): + return + + if not np.isfinite(center): + center = float(np.nanmedian(values)) + if not np.isfinite(sigma) or sigma <= 0: + sigma = float(np.nanstd(values)) + if not np.isfinite(sigma) or sigma <= 0: + return + if not np.isfinite(lower) or not np.isfinite(upper) or lower >= upper: + lower, upper = float(np.nanmin(values)), float(np.nanmax(values)) + if not np.isfinite(lower) or not np.isfinite(upper) or lower >= upper: + return + + x_values = np.linspace(lower, upper, 300) + y_values = np.exp(-0.5 * ((x_values - center) / sigma) ** 2) + y_max = y_values.max() if y_values.size else np.nan + if not np.isfinite(y_max) or y_max <= 0: + return + axis_top = ax.get_ylim()[1] + if not np.isfinite(axis_top) or axis_top <= 0: + axis_top = 1.0 + y_values = y_values / y_max * axis_top * 0.90 + + ax.plot(x_values, y_values, color='#c2410c', linewidth=1.5, label='Gaussian') + ax.axvline(center, color='#c2410c', linestyle='--', linewidth=1.0, label='Peak fit') + ax.set_ylim(0, max(axis_top, float(np.nanmax(y_values)) * 1.05)) + ax.legend(loc='best', fontsize=8, frameon=False) + + def _adjust_triangle_plot_layout(self, fig): + if not hasattr(fig, 'subplots_adjust'): + return + subplotpars = getattr(fig, 'subplotpars', None) + if subplotpars is None: + return + + fig.subplots_adjust( + left=max(subplotpars.left, 0.08), + bottom=max(subplotpars.bottom, 0.12), + right=min(subplotpars.right, 0.97), + top=min(subplotpars.top, 0.94), + wspace=subplotpars.wspace, + hspace=subplotpars.hspace, + ) def fit_LM(self): freekeys = list(self.bounds.keys()) boundarray = np.array([self.bounds[k] for k in freekeys]) + self._validate_flux_baseline_keys() # trim data around predicted transit/eclipse time if np.ndim(self.airmass) == 2: @@ -185,7 +3864,7 @@ def fit_LM(self): def lc2min_nneighbor(pars): for i in range(len(pars)): self.prior[freekeys[i]] = pars[i] - lightcurve = transit(self.time, self.prior) + lightcurve = self._transit_model(self.time, self.prior) detrended = self.data / lightcurve wf = weightedflux(detrended, self.gw, self.nearest) model = lightcurve * wf @@ -194,8 +3873,23 @@ def lc2min_nneighbor(pars): def lc2min_airmass(pars): for i in range(len(pars)): self.prior[freekeys[i]] = pars[i] - model = transit(self.time, self.prior) - model *= self.prior['a1'] * np.exp(self.prior['a2'] * self.airmass) + model = self._transit_model(self.time, self.prior) + model *= airmass_trend( + self.prior.get('a2', 0), + self.airmass, + reference=self._get_airmass_reference(), + ) + if self._has_free_flux_baseline(): + model *= get_flux_baseline(self.prior) + elif self._uses_fixed_flux_baseline(): + model *= get_flux_baseline(self.prior) + else: + model *= solve_flux_baseline( + model, + self.data, + self.dataerr, + mask=self._get_baseline_fit_mask(), + ) return ((self.data - model) / self.dataerr) ** 2 try: @@ -224,22 +3918,69 @@ def lc2min_airmass(pars): self.parameters = copy.deepcopy(self.prior) self.errors = {} + self.quantiles = {} for i, k in enumerate(freekeys): self.parameters[k] = res.x[i] self.errors[k] = 0 + self.quantiles[k] = [0, 0] + + self.sampled_keys = list(freekeys) + self.sample_bounds = copy.deepcopy(self.bounds) + self.sample_parameters = {k: self.parameters[k] for k in self.sampled_keys} + self.sample_errors = {k: self.errors[k] for k in self.sampled_keys} + self.sample_quantiles = {k: self.quantiles[k] for k in self.sampled_keys} self.create_fit_variables() def create_fit_variables(self): - self.phase = get_phase(self.time, self.parameters['per'], self.parameters['tmid']) - self.transit = transit(self.time, self.parameters) - self.time_upsample = np.linspace(min(self.time), max(self.time), 1000) - self.transit_upsample = transit(self.time_upsample, self.parameters) - self.phase_upsample = get_phase(self.time_upsample, self.parameters['per'], self.parameters['tmid']) - if self.mode == "ns": - self.parameters['a1'], self.errors['a1'] = mc_a1(self.parameters.get('a2', 0), self.errors.get('a2', 1e-6), - self.transit, self.airmass, self.data) + self.transit = self._transit_model(self.time, self.parameters) + self._apply_fixed_parameter_errors() + self._update_plot_geometry() + if np.ndim(self.airmass) != 2: + if self._has_free_flux_baseline(): + flux_scale = get_flux_baseline(self.parameters) + flux_scale_err = self.errors.get('a0', self.errors.get('a1', 0.0)) + elif self._uses_fixed_flux_baseline(): + flux_scale = get_flux_baseline(self.parameters) + flux_scale_err = self.errors.get( + 'a0', + self.errors.get('a1', self.fixed_parameter_errors.get('a0', 0.0)), + ) + elif self.mode == "ns": + flux_scale, flux_scale_err = mc_a1( + self.parameters.get('a2', 0), + self.errors.get('a2', 1e-6), + self.transit, + self.airmass, + self.data, + self.dataerr, + mask=self._get_baseline_fit_mask(), + ) + else: + systematics = self.transit * airmass_trend( + self.parameters.get('a2', 0), + self.airmass, + reference=self._get_airmass_reference(), + ) + flux_scale = solve_flux_baseline( + systematics, + self.data, + self.dataerr, + mask=self._get_baseline_fit_mask(), + ) + flux_scale_err = self.errors.get( + 'a0', + self.errors.get( + 'a1', + solve_flux_baseline_uncertainty( + systematics, + self.dataerr, + mask=self._get_baseline_fit_mask(), + ), + ), + ) + self._set_flux_baseline(flux_scale, flux_scale_err) if np.ndim(self.airmass) == 2: detrended = self.data / self.transit self.wf = weightedflux(detrended, self.gw, self.nearest) @@ -247,7 +3988,7 @@ def create_fit_variables(self): self.detrended = self.data / self.wf self.detrendederr = self.dataerr / self.wf else: - self.airmass_model = self.parameters['a1'] * np.exp(self.parameters.get('a2', 0) * self.airmass) + self.airmass_model = self._build_systematics_model(self.parameters) self.model = self.transit * self.airmass_model self.detrended = self.data / self.airmass_model self.detrendederr = self.dataerr / self.airmass_model @@ -272,116 +4013,485 @@ def create_fit_variables(self): tdur = (self.transit < 1).sum() * np.median(np.diff(np.sort(self.time))) # test for partial transit - newtime = np.linspace(self.parameters['tmid'] - 0.2, self.parameters['tmid'] + 0.2, 10000) - newtran = transit(newtime, self.parameters) - masktran = newtran < 1 - newdur = np.diff(newtime).mean() * masktran.sum() + newdur = transit_duration(self.parameters) + if not np.isfinite(newdur) or newdur <= 0: + newtime = np.linspace(self.parameters['tmid'] - 0.2, self.parameters['tmid'] + 0.2, 10000) + newtran = transit(newtime, self.parameters) + masktran = newtran < 1 + newdur = np.diff(newtime).mean() * masktran.sum() self.duration_measured = tdur self.duration_expected = newdur + def _finalize_ultranest_fit_results(self, bound_keys, sampled_keys, physical_from_sample_point): + self.sample_parameters = {} + self.sample_errors = {} + self.sample_quantiles = {} + self.errors = {} + self.quantiles = {} + self.parameters = copy.deepcopy(self.prior) + + ml_point = self.results['maximum_likelihood']['point'] + self.sample_bounds = self._get_sample_bounds(bound_keys, physical_from_sample_point(ml_point)) + self.ultranest_error_fallbacks = {} + weighted_points, weighted_logl = self._get_ultranest_weighted_sample_arrays() + + for i, key in enumerate(sampled_keys): + self.sample_parameters[key] = ml_point[i] + reported_error = self.results['posterior']['stdev'][i] + reported_quantiles = [ + self.results['posterior']['errlo'][i], + self.results['posterior']['errup'][i]] + if self._ultranest_error_needs_sample_fallback( + i, + ml_point[i], + reported_error, + points=weighted_points, + ): + fallback = self._loglike_neighborhood_uncertainty( + i, + ml_point[i], + points=weighted_points, + logl=weighted_logl, + ) + if fallback is not None: + fallback['reason'] = 'degenerate_posterior_summary' + else: + fallback = None + local_uncertainty = self._loglike_neighborhood_uncertainty( + i, + ml_point[i], + points=weighted_points, + logl=weighted_logl, + ) + if self._ultranest_error_is_inflated_relative_to_local_fit(reported_error, local_uncertainty): + fallback = local_uncertainty + fallback['reported_error'] = float(reported_error) + fallback['reason'] = 'posterior_summary_inflated_relative_to_local_fit' + if fallback is not None: + self.sample_errors[key] = fallback['error'] + self.sample_quantiles[key] = fallback['quantiles'] + self.ultranest_error_fallbacks[key] = fallback + else: + self.sample_errors[key] = reported_error + self.sample_quantiles[key] = reported_quantiles + + physical_ml = physical_from_sample_point(ml_point) + self.parameters.update(physical_ml) + + for bound_key, sampled_key in zip(bound_keys, sampled_keys): + if bound_key == 'inc' and sampled_key == 'b': + continue + self.errors[bound_key] = self.sample_errors[sampled_key] + self.quantiles[bound_key] = self.sample_quantiles[sampled_key] + + if 'inc' in bound_keys and 'b' in sampled_keys and weighted_points is not None: + bound_index = {key: index for index, key in enumerate(bound_keys)} + + def weighted_sample_values(key, default=0.0): + index = bound_index.get(key) + if index is not None and index < weighted_points.shape[1] and sampled_keys[index] != 'b': + return weighted_points[:, index] + value = physical_ml.get(key, self.prior.get(key, default)) + return np.full(weighted_points.shape[0], float(value), dtype=float) + + b_index = sampled_keys.index('b') + scale_values = { + 'ars': weighted_sample_values('ars', np.nan), + 'ecc': weighted_sample_values('ecc', 0.0), + 'omega': weighted_sample_values('omega', 0.0), + } + inc_samples = np.asarray( + inclination_from_impact_parameter(scale_values, weighted_points[:, b_index]), + dtype=float, + ) + center, std, quantiles = self._summarize_derived_parameter(inc_samples, physical_ml['inc']) + self.parameters['inc'] = center + self.errors['inc'] = std + self.quantiles['inc'] = quantiles + self._apply_fixed_parameter_errors() + + def extend_ultranest_fit(self, min_num_live_points=None, max_ncalls=None): + context = getattr(self, '_ultranest_resume_context', None) + if getattr(self, 'ns_type', None) != 'ultranest' or not isinstance(context, dict): + return False + + sampler = context.get('sampler') + if sampler is None: + return False + + run_kwargs = {"max_ncalls": int(max_ncalls if max_ncalls is not None else self.max_ncalls)} + if min_num_live_points is not None: + run_kwargs["min_num_live_points"] = int(min_num_live_points) + + self.results = run_reactive_sampler( + sampler, + run_kwargs=run_kwargs, + verbose=self.verbose, + ) + if getattr(self, 'ultranest_expanded_prior_warmstart_applied', False): + if not self._expanded_prior_warmstart_result_is_usable( + self.results, + len(context['sampled_keys']), + ): + return False + self._restore_expanded_prior_physical_likelihoods( + self.results, + context['loglike'], + len(context['sampled_keys']), + ) + self._finalize_ultranest_fit_results( + context['bound_keys'], + context['sampled_keys'], + context['physical_from_sample_point'], + ) + self.create_fit_variables() + return True + + def clear_ultranest_resume_state(self): + self._ultranest_resume_context = None + def fit_nested(self): - freekeys = list(self.bounds.keys()) - boundarray = np.array([self.bounds[k] for k in freekeys]) - bounddiff = np.diff(boundarray, 1).reshape(-1) + bound_keys = list(self.bounds.keys()) + sampled_keys = self._get_sampled_keys(bound_keys) + self._validate_flux_baseline_keys() + self.sampled_keys = list(sampled_keys) + self.sample_bounds = self._get_sample_bounds(bound_keys, self.prior) + self.impact_parameter_sampled_directly = self._uses_internal_impact_parameter() + + if len(set(self.sampled_keys)) != len(self.sampled_keys): + raise ValueError("Free-parameter labels must be unique after internal parameter transforms.") # alloc data for best fit + error + self.sample_parameters = {} + self.sample_errors = {} + self.sample_quantiles = {} self.errors = {} self.quantiles = {} self.parameters = copy.deepcopy(self.prior) + base_physical = dict(self.prior) + direct_sample_assignments = [ + (bound_key, index) + for index, (bound_key, sampled_key) in enumerate(zip(bound_keys, sampled_keys)) + if not (sampled_key == 'b' and bound_key == 'inc') + ] + impact_parameter_index = next( + ( + index + for index, (bound_key, sampled_key) in enumerate(zip(bound_keys, sampled_keys)) + if sampled_key == 'b' and bound_key == 'inc' + ), + None, + ) + time = self.time + data = np.asarray(self.data, dtype=float) + dataerr = np.asarray(self.dataerr, dtype=float) + data_shape = data.shape + dataerr_shape_matches = dataerr.shape == data_shape + finite_dataerr = np.isfinite(dataerr) & (dataerr > 0) if dataerr_shape_matches else False + observed_values_valid = ( + dataerr_shape_matches + and np.all(np.isfinite(data)) + and np.all(finite_dataerr) + ) + inverse_dataerr = np.zeros(data_shape, dtype=float) + baseline_weights = np.zeros(data_shape, dtype=float) + baseline_static_mask = np.isfinite(data) + if dataerr_shape_matches: + inverse_dataerr[finite_dataerr] = 1.0 / dataerr[finite_dataerr] + baseline_weights[finite_dataerr] = inverse_dataerr[finite_dataerr] ** 2 + baseline_static_mask &= np.isfinite(baseline_weights) & (baseline_weights > 0) + else: + baseline_static_mask &= False + + baseline_fit_mask = self._get_baseline_fit_mask() + if baseline_fit_mask is not None: + baseline_static_mask &= baseline_fit_mask + + centered_airmass = center_airmass(self.airmass, reference=self._get_airmass_reference()) + has_free_flux_baseline = self._has_free_flux_baseline() + uses_fixed_flux_baseline = self._uses_fixed_flux_baseline() + sampled_key_index = {key: index for index, key in enumerate(sampled_keys)} + sampled_a2_index = sampled_key_index.get('a2') + fixed_airmass_scale = None + if sampled_a2_index is None: + try: + fixed_a2 = float(base_physical.get('a2', 0.0)) + except (TypeError, ValueError): + fixed_a2 = np.nan + if not np.isfinite(fixed_a2): + observed_values_valid = False + elif fixed_a2 != 0.0: + fixed_airmass_scale = np.exp(fixed_a2 * centered_airmass) + + free_flux_baseline_index = next( + (sampled_key_index[key] for key in ('a0', 'a1') if key in sampled_key_index), + None, + ) + fixed_flux_baseline_value = None + if free_flux_baseline_index is None and uses_fixed_flux_baseline: + fixed_flux_baseline_value = get_flux_baseline(base_physical) + + duration_prior = self.duration_prior if isinstance(self.duration_prior, dict) else None + duration_prior_applied = bool(duration_prior and duration_prior.get('applied')) + try: + expected_duration = float(duration_prior.get('expected_duration', np.nan)) if duration_prior else np.nan + sigma_log_duration = float(duration_prior.get('sigma_log_duration', np.nan)) if duration_prior else np.nan + except (TypeError, ValueError): + expected_duration = np.nan + sigma_log_duration = np.nan + duration_prior_valid = ( + duration_prior_applied + and np.isfinite(expected_duration) + and expected_duration > 0 + and np.isfinite(sigma_log_duration) + and sigma_log_duration > 0 + ) + + def solve_flux_baseline_for_model(model): + mask = baseline_static_mask & np.isfinite(model) & (model != 0) + if not np.any(mask): + return fallback_flux_baseline() + + masked_model = model[mask] + masked_data = data[mask] + masked_weights = baseline_weights[mask] + denom = np.sum(masked_weights * masked_model ** 2) + + if not np.isfinite(denom) or denom <= 0: + ratio = masked_data / masked_model + ratio = ratio[np.isfinite(ratio)] + if ratio.size == 0: + return fallback_flux_baseline() + baseline = np.nanmedian(ratio) + return baseline if np.isfinite(baseline) else fallback_flux_baseline() + + baseline = np.sum(masked_weights * masked_data * masked_model) / denom + return baseline if np.isfinite(baseline) else fallback_flux_baseline() + + def physical_from_sample_point(sample_point): + physical = base_physical.copy() + for bound_key, index in direct_sample_assignments: + physical[bound_key] = sample_point[index] + if impact_parameter_index is not None: + impact_parameter = sample_point[impact_parameter_index] + physical['b'] = impact_parameter + physical['inc'] = float(inclination_from_impact_parameter(physical, impact_parameter)) + return physical + + def single_loglike(pars): + if not observed_values_valid: + return BAD_LOG_LIKELIHOOD + + physical = physical_from_sample_point(pars) + duration_loglike = 0.0 + if duration_prior_valid: + duration = transit_duration(physical) + if not np.isfinite(duration) or duration <= 0: + return BAD_LOG_LIKELIHOOD + duration_log_residual = np.log(duration / expected_duration) + duration_loglike = -0.5 * (duration_log_residual / sigma_log_duration) ** 2 + try: + model = np.asarray(self._transit_model(time, physical), dtype=float) + if sampled_a2_index is not None: + model *= np.exp(float(pars[sampled_a2_index]) * centered_airmass) + elif fixed_airmass_scale is not None: + model *= fixed_airmass_scale + + if free_flux_baseline_index is not None: + model *= pars[free_flux_baseline_index] + elif fixed_flux_baseline_value is not None: + model *= fixed_flux_baseline_value + elif has_free_flux_baseline: + model *= get_flux_baseline(physical) + else: + model *= solve_flux_baseline_for_model(model) + except Exception: + return BAD_LOG_LIKELIHOOD + + if model.shape != data_shape or not np.all(np.isfinite(model)): + return BAD_LOG_LIKELIHOOD + + residuals = (data - model) * inverse_dataerr + chi2 = np.sum(residuals * residuals) + logl = -0.5 * chi2 + duration_loglike + return float(logl) if np.isfinite(logl) else BAD_LOG_LIKELIHOOD + def loglike(pars): - # chi-squared - for i in range(len(pars)): - self.prior[freekeys[i]] = pars[i] - model = transit(self.time, self.prior) - model *= np.exp(self.prior['a2'] * self.airmass) - detrend = self.data / model # used to estimate a1 - model *= np.median(detrend) - return -0.5 * np.sum(((self.data - model) / self.dataerr) ** 2) + pars_array = np.asarray(pars, dtype=float) + if pars_array.ndim == 2: + return np.fromiter( + (single_loglike(row) for row in pars_array), + dtype=float, + count=pars_array.shape[0], + ) + return single_loglike(pars_array) + + prior_boundarray = np.array([self.bounds[k] for k in bound_keys], dtype=float) + prior_lower_bounds = prior_boundarray[:, 0] + prior_bound_widths = prior_boundarray[:, 1] - prior_lower_bounds + prior_bound_index = {key: index for index, key in enumerate(bound_keys)} + prior_inc_index = prior_bound_index.get('inc') + + def prior_values_for(sample_points, key, default): + if key in prior_bound_index: + return sample_points[:, prior_bound_index[key]] + value = np.asarray(base_physical.get(key, default), dtype=float) + if value.shape == (): + return np.full(sample_points.shape[0], float(value), dtype=float) + return np.broadcast_to(value, (sample_points.shape[0],)).astype(float) + + def prior_impact_upper_bounds(sample_points): + rprs = prior_values_for(sample_points, 'rprs', np.nan) + grazing_upper = np.where(np.isfinite(rprs) & (rprs >= 0), 1.0 + rprs, np.nan) + + ars = prior_values_for(sample_points, 'ars', np.nan) + ecc = prior_values_for(sample_points, 'ecc', 0.0) + omega = np.deg2rad(prior_values_for(sample_points, 'omega', 0.0)) + denom = 1.0 + ecc * np.sin(omega) + denom = np.where(np.isclose(denom, 0.0), np.finfo(float).eps, denom) + scale_upper = ars * (1.0 - ecc ** 2) / denom + + valid_grazing = np.isfinite(grazing_upper) & (grazing_upper > 0) + valid_scale = np.isfinite(scale_upper) & (scale_upper > 0) + upper = np.full(sample_points.shape[0], 1.0, dtype=float) + + both_valid = valid_grazing & valid_scale + upper[both_valid] = np.minimum(grazing_upper[both_valid], scale_upper[both_valid]) + upper[valid_grazing & ~valid_scale] = grazing_upper[valid_grazing & ~valid_scale] + upper[valid_scale & ~valid_grazing] = scale_upper[valid_scale & ~valid_grazing] + return np.maximum(0.0, upper) def prior_transform(upars): - # transform unit cube to prior volume - return boundarray[:, 0] + bounddiff * upars + upars_array = np.asarray(upars, dtype=float) + sample_points = prior_lower_bounds + prior_bound_widths * upars_array + if not self.impact_parameter_sampled_directly or prior_inc_index is None: + return sample_points + + if upars_array.ndim == 2: + upper_bounds = prior_impact_upper_bounds(sample_points) + sample_points[:, prior_inc_index] = upper_bounds * upars_array[:, prior_inc_index] + return sample_points + + upper_bound = prior_impact_upper_bounds(sample_points.reshape(1, -1))[0] + sample_points[prior_inc_index] = upper_bound * upars_array[prior_inc_index] + return sample_points + + self.ns_type = 'ultranest' + warmstart_problem, warmstart_skip_reason = self._build_expanded_prior_warmstart_problem( + bound_keys, + sampled_keys, + loglike, + prior_transform, + ) + self.ultranest_expanded_prior_warmstart_attempted = ( + getattr(self, 'ultranest_warmstart_source', None) is not None + ) + if warmstart_problem is not None: + test = ReactiveNestedSampler( + warmstart_problem['param_names'], + warmstart_problem['loglike'], + warmstart_problem['transform'], + vectorized=warmstart_problem['vectorized'], + ) + self.ultranest_expanded_prior_warmstart_applied = True + self.ultranest_expanded_prior_warmstart_note = ( + "Applied a corrected expanded-prior UltraNest warm start using " + f"{self.ultranest_expanded_prior_warmstart_source_sample_count} previous " + "weighted posterior sample(s), with " + f"{100.0 * warmstart_problem['full_prior_fraction']:.0f}% of the auxiliary " + "prior reserved for direct exploration of the full expanded prior." + ) + else: + test = ReactiveNestedSampler(sampled_keys, loglike, prior_transform, vectorized=True) + self.ultranest_expanded_prior_warmstart_applied = False + self.ultranest_expanded_prior_warmstart_note = warmstart_skip_reason + + run_kwargs = {"max_ncalls": int(self.max_ncalls)} + if self.ultranest_min_num_live_points is not None: + run_kwargs["min_num_live_points"] = int(self.ultranest_min_num_live_points) try: - self.ns_type = 'ultranest' - test = ReactiveNestedSampler(freekeys, loglike, prior_transform) + self.results = run_reactive_sampler( + test, + run_kwargs=run_kwargs, + verbose=self.verbose, + ) + if ( + warmstart_problem is not None + and not self._expanded_prior_warmstart_result_is_usable( + self.results, + len(sampled_keys), + ) + ): + raise ValueError( + "the corrected warm-start result did not contain usable posterior samples" + ) + if warmstart_problem is not None: + self._restore_expanded_prior_physical_likelihoods( + self.results, + loglike, + len(sampled_keys), + ) + except Exception as exc: + if warmstart_problem is None: + raise + self.ultranest_expanded_prior_warmstart_applied = False + self.ultranest_expanded_prior_warmstart_note = ( + "Corrected expanded-prior warm start failed; reran from the full expanded " + f"prior instead ({type(exc).__name__}: {exc})." + ) + if self.verbose: + print(f"WARNING: {self.ultranest_expanded_prior_warmstart_note}") + test = ReactiveNestedSampler( + sampled_keys, + loglike, + prior_transform, + vectorized=True, + ) + self.results = run_reactive_sampler( + test, + run_kwargs=run_kwargs, + verbose=self.verbose, + ) + + if self.keep_ultranest_sampler: + self._ultranest_resume_context = { + 'sampler': test, + 'bound_keys': list(bound_keys), + 'sampled_keys': list(sampled_keys), + 'physical_from_sample_point': physical_from_sample_point, + 'loglike': loglike, + } + else: + self._ultranest_resume_context = None + self._finalize_ultranest_fit_results(bound_keys, sampled_keys, physical_from_sample_point) - noop = lambda *args, **kwargs: None - if self.verbose is True: - self.results = test.run(max_ncalls=int(self.max_ncalls)) - else: - self.results = test.run(max_ncalls=int(self.max_ncalls), show_status=False, viz_callback=noop) - - for i, key in enumerate(freekeys): - self.parameters[key] = self.results['maximum_likelihood']['point'][i] - self.errors[key] = self.results['posterior']['stdev'][i] - self.quantiles[key] = [ - self.results['posterior']['errlo'][i], - self.results['posterior']['errup'][i]] - except NameError: - self.ns_type = 'dynesty' - dsampler = dynesty.DynamicNestedSampler(loglike, prior_transform, ndim=len(freekeys), - bound='multi', sample='unif') - dsampler.run_nested(maxcall=int(1e5), dlogz_init=0.05, - maxbatch=10, nlive_batch=100, print_progress=self.verbose) - self.results = dsampler.results - - tests = [copy.deepcopy(self.prior) for i in range(5)] - - # Derive kernel density estimate for best fit - weights = np.exp(self.results.logwt - self.results.logz[-1]) - samples = self.results['samples'] - logvol = self.results['logvol'] - wt_kde = gaussian_kde(resample_equal(-logvol, weights)) # KDE - logvol_grid = np.linspace(logvol[0], logvol[-1], 1000) # resample - wt_grid = wt_kde.pdf(-logvol_grid) # evaluate KDE PDF - self.weights = np.interp(-logvol, -logvol_grid, wt_grid) # interpolate - - # errors + final values - mean, cov = dynesty.utils.mean_and_cov(self.results.samples, weights) - mean2, cov2 = dynesty.utils.mean_and_cov(self.results.samples, self.weights) - for i in range(len(freekeys)): - self.errors[freekeys[i]] = cov[i, i] ** 0.5 - tests[0][freekeys[i]] = mean[i] - tests[1][freekeys[i]] = mean2[i] - - counts, bins = np.histogram(samples[:, i], bins=100, weights=weights) - mi = np.argmax(counts) - tests[4][freekeys[i]] = bins[mi] + 0.5 * np.mean(np.diff(bins)) - - # finds median and +- 2sigma, will vary from mode if non-gaussian - self.quantiles[freekeys[i]] = dynesty.utils.quantile(self.results.samples[:, i], [0.025, 0.5, 0.975], - weights=weights) - tests[2][freekeys[i]] = self.quantiles[freekeys[i]][1] - - # find minimum near weighted mean - mask = (samples[:, 0] < self.parameters[freekeys[0]] + 2 * self.errors[freekeys[0]]) & ( - samples[:, 0] > self.parameters[freekeys[0]] - 2 * self.errors[freekeys[0]]) - bi = np.argmin(self.weights[mask]) - - for i in range(len(freekeys)): - tests[3][freekeys[i]] = samples[mask][bi, i] - # tests[4][freekeys[i]] = np.average(samples[mask][:, i], weights=self.weights[mask], axis=0) - - # find best fit from chi2 minimization - chis = [] - for i in range(len(tests)): - lightcurve = transit(self.time, tests[i]) - tests[i]['a1'] = mc_a1(tests[i].get('a2', 0), self.errors.get('a2', 1e-6), - lightcurve, self.airmass, self.data)[0] - airmass = tests[i]['a1'] * np.exp(tests[i].get('a2', 0) * self.airmass) - residuals = self.data - (lightcurve * airmass) - chis.append(np.sum(residuals ** 2)) - - mi = np.argmin(chis) - self.parameters = copy.deepcopy(tests[mi]) + if not self.sample_parameters: + self.sample_parameters = { + key: self.parameters.get(key, self.sample_parameters.get(key)) + for key in self.sampled_keys + } + for bound_key, sampled_key in zip(bound_keys, sampled_keys): + if sampled_key not in self.sample_errors and bound_key in self.errors: + self.sample_errors[sampled_key] = self.errors[bound_key] + if sampled_key not in self.sample_quantiles and bound_key in self.quantiles: + self.sample_quantiles[sampled_key] = self.quantiles[bound_key] # final model self.create_fit_variables() - def plot_bestfit(self, title="", bin_dt=30. / (60 * 24), zoom=False, phase=True): + def plot_bestfit( + self, + title="", + bin_dt=30. / (60 * 24), + zoom=False, + phase=True, + show_flux_baseline_label=True, + show_model_uncertainty=False, + show_baseline_uncertainty=False, + ): f = plt.figure(figsize=(9, 6)) f.subplots_adjust(top=0.92, bottom=0.09, left=0.14, right=0.98, hspace=0) ax_lc = plt.subplot2grid((4, 5), (0, 0), colspan=5, rowspan=3) @@ -393,37 +4503,60 @@ def plot_bestfit(self, title="", bin_dt=30. / (60 * 24), zoom=False, phase=True) axs[0].grid(True, ls='--') rprs2 = self.parameters['rprs'] ** 2 - rprs2err = 2 * self.parameters['rprs'] * self.errors['rprs'] - lclabel1 = r"$R^{2}_{p}/R^{2}_{s}$ = %s $\pm$ %s" % ( - str(round_to_2(rprs2, rprs2err)), - str(round_to_2(rprs2err)) + rprs_error_for_depth = self._combined_rprs_uncertainty_for_reporting() + rprs2err = 2 * self.parameters['rprs'] * rprs_error_for_depth + rprs_prior_marker = " (Prior)" if getattr(self, 'rprs_prior_fallback_applied', False) else "" + rprs2_text, rprs2err_text = format_value_error_for_plot(rprs2, rprs2err) + lclabel1 = r"$(R_{p}/R_{s})^{2}$ = %s $\pm$ %s%s" % ( + rprs2_text, + rprs2err_text, + rprs_prior_marker, ) + tmid_error_for_plot = self._model_data_uncertainty_for_reporting('tmid') + if not np.isfinite(tmid_error_for_plot): + tmid_error_for_plot = self.errors.get('tmid', 0) + tmid_text, tmid_error_text = format_value_error_for_plot( + self.parameters['tmid'], + tmid_error_for_plot, + ) lclabel2 = r"$T_{mid}$ = %s $\pm$ %s BJD$_{TDB}$" % ( - str(round_to_2(self.parameters['tmid'], self.errors.get('tmid', 0))), - str(round_to_2(self.errors.get('tmid', 0))) + tmid_text, + tmid_error_text, ) lclabel = lclabel1 + "\n" + lclabel2 + if show_flux_baseline_label and 'a0' in self.parameters: + a0_text, a0_error_text = format_value_error_for_plot( + self.parameters['a0'], + self.errors.get('a0', 0), + ) + lclabel3 = r"$a_0$ = %s $\pm$ %s" % ( + a0_text, + a0_error_text, + ) + lclabel += "\n" + lclabel3 if zoom: axs[0].set_ylim([1 - 1.25 * self.parameters['rprs'] ** 2, 1 + 0.5 * self.parameters['rprs'] ** 2]) else: if phase: axs[0].errorbar(self.phase, self.detrended, yerr=np.std(self.residuals) / np.median(self.data), - ls='none', marker='.', color='black', zorder=1, alpha=0.2) + ls='none', marker='.', color='black', ecolor='0.72', + elinewidth=1.0, zorder=1, alpha=1.0) else: axs[0].errorbar(self.time, self.detrended, yerr=np.std(self.residuals) / np.median(self.data), - ls='none', marker='.', color='black', zorder=1, alpha=0.2) + ls='none', marker='.', color='black', ecolor='0.72', + elinewidth=1.0, zorder=1, alpha=1.0) if phase: si = np.argsort(self.phase) bt2, br2, _ = time_bin(self.phase[si] * self.parameters['per'], self.residuals[si] / np.median(self.data) * 1e2, bin_dt) - axs[1].plot(self.phase, self.residuals / np.median(self.data) * 1e2, 'k.', alpha=0.2, + axs[1].plot(self.phase, self.residuals / np.median(self.data) * 1e2, 'k.', alpha=1.0, label=r'$\sigma$ = {:.2f} %'.format(np.std(self.residuals / np.median(self.data) * 1e2))) axs[1].plot(bt2 / self.parameters['per'], br2, 'bs', alpha=1, zorder=2) - axs[1].set_xlim([min(self.phase), max(self.phase)]) + axs[1].set_xlim([min(self.phase_upsample), max(self.phase_upsample)]) axs[1].set_xlabel("Phase", fontsize=14) si = np.argsort(self.phase) @@ -432,23 +4565,55 @@ def plot_bestfit(self, title="", bin_dt=30. / (60 * 24), zoom=False, phase=True) marker='s') # axs[0].plot(self.phase[si], self.transit[si], 'r-', zorder=3, label=lclabel) sii = np.argsort(self.phase_upsample) + if show_baseline_uncertainty: + self._plot_baseline_model_uncertainty( + axs[0], + self.phase_upsample, + self.time_upsample, + sii, + label='_nolegend_', + ) + if show_model_uncertainty: + self._plot_transit_model_uncertainty( + axs[0], + self.phase_upsample, + self.time_upsample, + sii, + label='_nolegend_', + ) axs[0].plot(self.phase_upsample[sii], self.transit_upsample[sii], 'r-', zorder=3, label=lclabel) - axs[0].set_xlim([min(self.phase), max(self.phase)]) + axs[0].set_xlim([min(self.phase_upsample), max(self.phase_upsample)]) axs[0].set_xlabel("Phase ", fontsize=14) else: bt, br, _ = time_bin(self.time, self.residuals / np.median(self.data) * 1e2, bin_dt) - axs[1].plot(self.time, self.residuals / np.median(self.data) * 1e2, 'k.', alpha=0.2, + axs[1].plot(self.time, self.residuals / np.median(self.data) * 1e2, 'k.', alpha=1.0, label=r'$\sigma$ = {:.2f} %'.format(np.std(self.residuals / np.median(self.data) * 1e2))) axs[1].plot(bt, br, 'bs', alpha=1, zorder=2, label=r'$\sigma$ = {:.2f} %'.format(np.std(br))) - axs[1].set_xlim([min(self.time), max(self.time)]) + axs[1].set_xlim([min(self.time_upsample), max(self.time_upsample)]) axs[1].set_xlabel("Time [day]", fontsize=14) bt, bf, bs = time_bin(self.time, self.detrended, bin_dt) si = np.argsort(self.time) sii = np.argsort(self.time_upsample) axs[0].errorbar(bt, bf, yerr=bs, alpha=1, zorder=2, color='blue', ls='none', marker='s') + if show_baseline_uncertainty: + self._plot_baseline_model_uncertainty( + axs[0], + self.time_upsample, + self.time_upsample, + sii, + label='_nolegend_', + ) + if show_model_uncertainty: + self._plot_transit_model_uncertainty( + axs[0], + self.time_upsample, + self.time_upsample, + sii, + label='_nolegend_', + ) axs[0].plot(self.time_upsample[sii], self.transit_upsample[sii], 'r-', zorder=3, label=lclabel) - axs[0].set_xlim([min(self.time), max(self.time)]) + axs[0].set_xlim([min(self.time_upsample), max(self.time_upsample)]) axs[0].set_xlabel("Time [day]", fontsize=14) axs[0].get_xaxis().set_visible(False) @@ -458,89 +4623,87 @@ def plot_bestfit(self, title="", bin_dt=30. / (60 * 24), zoom=False, phase=True) axs[1].grid(True, ls='--', axis='y') return f, axs - def plot_triangle(self): - if self.ns_type == 'ultranest': - ranges = [] - mask1 = np.ones(len(self.results['weighted_samples']['logl']), dtype=bool) - mask2 = np.ones(len(self.results['weighted_samples']['logl']), dtype=bool) - mask3 = np.ones(len(self.results['weighted_samples']['logl']), dtype=bool) - titles = [] - labels = [] - flabels = { - 'rprs': r'R$_{p}$/R$_{s}$', - 'per': r'Period [day]', - 'tmid': r'T$_{mid}$', - 'ars': r'a/R$_{s}$', - 'inc': r'Inc. [deg]', - 'u1': r'u$_1$', - 'fpfs': r'F$_{p}$/F$_{s}$', - 'omega': r'$\omega$ [deg]', - 'mplanet': r'M$_{p}$ [M$_{\oplus}$]', - 'mstar': r'M$_{s}$ [M$_{\odot}$]', - 'ecc': r'$e$', - 'c0': r'$c_0$', - 'c1': r'$c_1$', - 'c2': r'$c_2$', - 'c3': r'$c_3$', - 'c4': r'$c_4$', - 'a0': r'$a_0$', - 'a1': r'$a_1$', - 'a2': r'$a_2$' - } - for i, key in enumerate(self.quantiles): - labels.append(flabels.get(key, key)) - titles.append(f"{self.parameters[key]:.5f} +- {self.errors[key]:.5f}") - ranges.append([ - self.parameters[key] - 5 * self.errors[key], - self.parameters[key] + 5 * self.errors[key] - ]) - - if key == 'a2' or key == 'a1': - continue + def plot_triangle(self, plot_title=None, zoom_sigma=None): + payload = self._get_triangle_plot_payload() + if zoom_sigma is not None: + payload = dict(payload) + payload['ranges'] = self._triangle_plot_sigma_window_ranges(payload, zoom_sigma) + payload = self._recenter_triangle_plot_payload_for_visible_ranges(payload) + + chi2 = payload['display_logl'] * -2 + parameter_count = max(1, len(payload['sampled_keys'])) + fig_size = max(9.0, 2.35 * parameter_count) + mask1 = np.ones(len(chi2), dtype=bool) + mask2 = np.ones(len(chi2), dtype=bool) + mask3 = np.ones(len(chi2), dtype=bool) + + for i, key in enumerate(payload['sampled_keys']): + if key in ('a0', 'a1', 'a2'): + continue + + center = payload['mask_centers'][i] + error = payload['mask_errors'][i] + if not np.isfinite(center) or not np.isfinite(error) or error <= 0: + continue + + values = payload['mask_values'][:, i] + mask3 = mask3 & (values > (center - 3 * error)) & (values < (center + 3 * error)) + mask1 = mask1 & (values > (center - error)) & (values < (center + error)) + mask2 = mask2 & (values > (center - 2 * error)) & (values < (center + 2 * error)) + + if not np.any(mask1): + mask1 = np.ones(len(chi2), dtype=bool) + if not np.any(mask2): + mask2 = np.ones(len(chi2), dtype=bool) + if not np.any(mask3): + mask3 = np.ones(len(chi2), dtype=bool) + + label_kwargs = { + 'labelpad': 10, + 'fontsize': 10, + } + title_kwargs = { + 'loc': 'left', + 'pad': 4, + 'fontsize': 11, + } - mask3 = mask3 & \ - (self.results['weighted_samples']['points'][:, i] > (self.parameters[key] - 3 * self.errors[key])) & \ - (self.results['weighted_samples']['points'][:, i] < (self.parameters[key] + 3 * self.errors[key])) - - mask1 = mask1 & \ - (self.results['weighted_samples']['points'][:, i] > (self.parameters[key] - self.errors[key])) & \ - (self.results['weighted_samples']['points'][:, i] < (self.parameters[key] + self.errors[key])) - - mask2 = mask2 & \ - (self.results['weighted_samples']['points'][:, i] > (self.parameters[key] - 2 * self.errors[key])) & \ - (self.results['weighted_samples']['points'][:, i] < (self.parameters[key] + 2 * self.errors[key])) - - chi2 = self.results['weighted_samples']['logl'] * -2 - fig = corner(self.results['weighted_samples']['points'], - labels=labels, - bins=int(np.sqrt(self.results['samples'].shape[0])), - range=ranges, - # quantiles=(0.1, 0.84), - plot_contours=True, - levels=[np.percentile(chi2[mask1], 95), np.percentile(chi2[mask2], 95), - np.percentile(chi2[mask3], 95)], - plot_density=False, - titles=titles, - data_kwargs={ - 'c': chi2, - 'vmin': np.percentile(chi2[mask3], 1), - 'vmax': np.percentile(chi2[mask3], 95), - 'cmap': 'viridis' - }, - label_kwargs={ - 'labelpad': 15, - }, - hist_kwargs={ - 'color': 'black', - } - ) - else: - fig, axs = dynesty.plotting.cornerplot(self.results, labels=list(self.bounds.keys()), - quantiles_2d=[0.4, 0.85], - smooth=0.015, show_titles=True, use_math_text=True, title_fmt='.2e', - hist2d_kwargs={ 'fill_contours': False}) - dynesty.plotting.cornerpoints(self.results, labels=list(self.bounds.keys()), - fig=[fig, axs[1:, :-1]], plot_kwargs={'alpha': 0.1, 'zorder': 1, }) + fig = corner(payload['display_points'], + labels=payload['labels'], + bins=int(np.sqrt(payload['display_points'].shape[0])), + range=payload['ranges'], + weights=payload['display_weights'], + plot_contours=True, + levels=self._triangle_contour_levels(chi2, mask1, mask2, mask3), + plot_density=False, + titles=payload['titles'], + truths=payload['truths'], + data_kwargs={ + 'c': chi2, + 'vmin': np.percentile(chi2[mask3], 1), + 'vmax': np.percentile(chi2[mask3], 95), + 'cmap': 'viridis', + 's': 1.6, + 'alpha': 0.38, + }, + label_kwargs=label_kwargs, + title_kwargs=title_kwargs, + hist_kwargs={ + 'color': 'black', + } + ) + if hasattr(fig, 'set_size_inches'): + fig.set_size_inches(fig_size, fig_size, forward=True) + if plot_title and hasattr(fig, 'suptitle'): + fig.suptitle(plot_title, fontsize=13, y=0.99) + self._overlay_single_parameter_triangle_gaussian(fig, payload) + self._adjust_triangle_plot_layout(fig) + self._overlay_triangle_plot_geometry_histograms( + fig, + payload, + title_kwargs=title_kwargs, + label_kwargs=label_kwargs, + ) return fig # simultaneously fit multiple data sets with global and local parameters @@ -663,9 +4826,14 @@ def loglike(pars): # compute model model = transit(self.lc_data[i]['time'], self.lc_data[i]['priors']) - model *= np.exp(self.lc_data[i]['priors']['a2']*self.lc_data[i]['airmass']) - detrend = self.lc_data[i]['flux']/model - model *= np.mean(detrend) + model *= airmass_trend( + self.lc_data[i]['priors'].get('a2', 0), + self.lc_data[i]['airmass'], + ) + if has_explicit_flux_baseline(self.global_bounds) or has_explicit_flux_baseline(self.local_bounds[i]): + model *= get_flux_baseline(self.lc_data[i]['priors']) + else: + model *= solve_flux_baseline(model, self.lc_data[i]['flux'], self.lc_data[i]['ferr']) # add to chi2 chi2 += np.sum( ((self.lc_data[i]['flux']-model)/self.lc_data[i]['ferr'])**2 ) @@ -679,11 +4847,12 @@ def loglike(pars): #clean_name = self.lc_data[n].get('name', n).replace(' ','_').replace('(','').replace(')','').replace('[','').replace(']','').replace('-','_').split('-')[0] freekeys.append(f"local_{k}_{n}") - noop = lambda *args, **kwargs: None - if self.verbose: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=1e6, show_status=True) - else: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=1e6, show_status=False, viz_callback=noop) + sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform) + self.results = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": int(1e6)}, + verbose=self.verbose, + ) self.quantiles = {} self.errors = {} @@ -734,18 +4903,43 @@ def loglike(pars): local_rprs.append(self.lc_data[n]['priors'][k]) local_rprs_err.append(self.lc_data[n]['errors'][k]) - # solve for a1 + # solve for the local baseline flux scale model = transit(self.lc_data[n]['time'], self.lc_data[n]['priors']) - airmass = np.exp(self.lc_data[n]['airmass']*self.lc_data[n]['priors']['a2']) - detrend = self.lc_data[n]['flux']/(model*airmass) - self.lc_data[n]['priors']['a1'] = np.mean(detrend) - self.lc_data[n]['residuals'] = self.lc_data[n]['flux'] - model*airmass*self.lc_data[n]['priors']['a1'] - self.lc_data[n]['detrend'] = self.lc_data[n]['flux']/(airmass*self.lc_data[n]['priors']['a1']) + airmass = airmass_trend( + self.lc_data[n]['priors'].get('a2', 0), + self.lc_data[n]['airmass'], + ) + if has_explicit_flux_baseline(self.global_bounds) or has_explicit_flux_baseline(self.local_bounds[n]): + flux_scale = get_flux_baseline(self.lc_data[n]['priors']) + flux_scale_err = self.lc_data[n]['errors'].get('a0', self.lc_data[n]['errors'].get('a1', 0)) + else: + flux_scale = solve_flux_baseline(model * airmass, self.lc_data[n]['flux'], self.lc_data[n]['ferr']) + flux_scale_err = solve_flux_baseline_uncertainty(model * airmass, self.lc_data[n]['ferr']) + self.lc_data[n]['priors']['a0'] = flux_scale + self.lc_data[n]['priors']['a1'] = flux_scale + self.lc_data[n]['errors']['a0'] = flux_scale_err + self.lc_data[n]['errors']['a1'] = flux_scale_err + self.lc_data[n]['residuals'] = self.lc_data[n]['flux'] - model * airmass * flux_scale + self.lc_data[n]['detrend'] = self.lc_data[n]['flux'] / (airmass * flux_scale) # phase - self.lc_data[n]['phase'] = get_phase(self.lc_data[n]['time'], self.lc_data[n]['priors']['per'], self.lc_data[n]['priors']['tmid']) - self.lc_data[n]['time_upsample'] = np.linspace(min(self.lc_data[n]['time']), max(self.lc_data[n]['time']), 1000) - self.lc_data[n]['phase_upsample'] = get_phase(self.lc_data[n]['time_upsample'], self.lc_data[n]['priors']['per'], self.lc_data[n]['priors']['tmid']) + plot_time_range = normalize_time_range(self.lc_data[n].get('plot_time_range')) + if plot_time_range is None: + plot_time_range = normalize_time_range(self.lc_data[n]['time']) + self.lc_data[n]['plot_time_range'] = plot_time_range + self.lc_data[n]['phase'] = get_plot_phase( + self.lc_data[n]['time'], + self.lc_data[n]['priors']['per'], + self.lc_data[n]['priors']['tmid'], + plot_time_range, + ) + self.lc_data[n]['time_upsample'] = np.linspace(plot_time_range[0], plot_time_range[1], 1000) + self.lc_data[n]['phase_upsample'] = get_plot_phase( + self.lc_data[n]['time_upsample'], + self.lc_data[n]['priors']['per'], + self.lc_data[n]['priors']['tmid'], + plot_time_range, + ) self.lc_data[n]['transit_upsample'] = transit(self.lc_data[n]['time_upsample'], self.lc_data[n]['priors']) # create an average value from all the local fits, used for plotting final best fit @@ -783,8 +4977,11 @@ def plot_bestfits(self): nmarker = next(markers) model = transit(self.lc_data[i]['time'], self.lc_data[i]['priors']) - airmass = np.exp(self.lc_data[i]['airmass']*self.lc_data[i]['priors']['a2']) - detrend = self.lc_data[i]['flux']/(model*airmass) + airmass = airmass_trend( + self.lc_data[i]['priors'].get('a2', 0), + self.lc_data[i]['airmass'], + ) + detrend = self.lc_data[i]['flux'] / (model * airmass) if ax.ndim == 1: ax[i].axis('on') @@ -851,14 +5048,19 @@ def plot_bestfit(self, title="", bin_dt=30./(60*24), alpha=0.05, ylim_sigma=5, p rprs2 = self.lc_data[0]['priors']['rprs']**2 rprs2err = 2*self.lc_data[0]['priors']['rprs']*self.lc_data[0]['errors']['rprs'] - lclabel1 = r"$R^{2}_{p}/R^{2}_{s}$ = %s $\pm$ %s" %( - str(round_to_2(rprs2, rprs2err)), - str(round_to_2(rprs2err)) + rprs2_text, rprs2err_text = format_value_error_for_plot(rprs2, rprs2err) + lclabel1 = r"$(R_{p}/R_{s})^{2}$ = %s $\pm$ %s" %( + rprs2_text, + rprs2err_text, ) + tmid_text, tmid_error_text = format_value_error_for_plot( + self.parameters['tmid'], + self.errors.get('tmid',0), + ) lclabel2 = r"$T_{mid}$ = %s $\pm$ %s BJD$_{TDB}$" %( - str(round_to_2(self.parameters['tmid'], self.errors.get('tmid',0))), - str(round_to_2(self.errors.get('tmid',0))) + tmid_text, + tmid_error_text, ) lclabel = lclabel1 + "\n" + lclabel2 @@ -872,6 +5074,7 @@ def plot_bestfit(self, title="", bin_dt=30./(60*24), alpha=0.05, ylim_sigma=5, p alldata = { 'time': [], + 'phase': [], 'flux': [], 'detrend': [], 'ferr': [], @@ -882,12 +5085,13 @@ def plot_bestfit(self, title="", bin_dt=30./(60*24), alpha=0.05, ylim_sigma=5, p ncolor = next(colors) nmarker = next(markers) alldata['time'].extend(self.lc_data[n]['time'].tolist()) + alldata['phase'].extend(self.lc_data[n]['phase'].tolist()) alldata['detrend'].extend(self.lc_data[n]['detrend'].tolist()) alldata['flux'].extend(self.lc_data[n]['flux'].tolist()) alldata['ferr'].extend(self.lc_data[n]['ferr'].tolist()) alldata['residuals'].extend(self.lc_data[n]['residuals'].tolist()) - phase = get_phase(self.lc_data[n]['time'], self.parameters['per'], self.lc_data[n]['priors']['tmid']) + phase = self.lc_data[n]['phase'] si = np.argsort(phase) #bt2, br2, _ = time_bin(phase[si]*self.parameters['per'], self.lc_data[n]['residuals'][si]/np.median(self.lc_data[n]['flux'])*1e2, bin_dt) @@ -909,8 +5113,8 @@ def plot_bestfit(self, title="", bin_dt=30./(60*24), alpha=0.05, ylim_sigma=5, p label=r'{}: {:.2f} %'.format(self.lc_data[n].get('name',''),np.std(self.lc_data[n]['residuals']/np.median(self.lc_data[n]['flux'])*1e2))) # replace min and max for upsampled lc model - minp = min(minp, min(phase)) - maxp = max(maxp, max(phase)) + minp = min(minp, min(self.lc_data[n]['phase_upsample'])) + maxp = max(maxp, max(self.lc_data[n]['phase_upsample'])) min_std = min(min_std, np.std(self.lc_data[n]['residuals']/np.median(self.lc_data[n]['flux']))) # plot individual best fit models @@ -921,7 +5125,7 @@ def plot_bestfit(self, title="", bin_dt=30./(60*24), alpha=0.05, ylim_sigma=5, p for k in alldata.keys(): alldata[k] = np.array(alldata[k]) - phase = get_phase(alldata['time'], self.parameters['per'], self.lc_data[n]['priors']['tmid']) + phase = alldata['phase'] si = np.argsort(phase) bt, br, _ = time_bin(phase[si]*self.parameters['per'], alldata['residuals'][si]/np.median(alldata['flux']), 2*bin_dt) bt, bf, bs = time_bin(phase[si]*self.parameters['per'], alldata['detrend'][si], 2*bin_dt) @@ -934,12 +5138,10 @@ def plot_bestfit(self, title="", bin_dt=30./(60*24), alpha=0.05, ylim_sigma=5, p axs[1].plot(bt/self.parameters['per'],br*1e2,color='white',ls='none',marker='o',ms=11,markeredgecolor='black') # best fit model - self.time_upsample = np.linspace(minp*self.parameters['per']+self.parameters['tmid'], - maxp*self.parameters['per']+self.parameters['tmid'], 10000) + self.phase_upsample = np.linspace(minp, maxp, 10000) + self.time_upsample = self.parameters['tmid'] + self.phase_upsample * self.parameters['per'] self.transit_upsample = transit(self.time_upsample, self.parameters) - self.phase_upsample = get_phase(self.time_upsample, self.parameters['per'], self.parameters['tmid']) - sii = np.argsort(self.phase_upsample) - axs[0].plot(self.phase_upsample[sii], self.transit_upsample[sii], 'r-', zorder=3, label=lclabel, lw=3) + axs[0].plot(self.phase_upsample, self.transit_upsample, 'r-', zorder=3, label=lclabel, lw=3) # set up axes limits axs[0].set_xlim([min(self.phase_upsample), max(self.phase_upsample)]) @@ -952,8 +5154,8 @@ def plot_bestfit(self, title="", bin_dt=30./(60*24), alpha=0.05, ylim_sigma=5, p # compute average min and max for all the data mins = []; maxs = [] for n in range(len(self.lc_data)): - mins.append(min(self.lc_data[n]['phase'])) - maxs.append(max(self.lc_data[n]['phase'])) + mins.append(min(self.lc_data[n]['phase_upsample'])) + maxs.append(max(self.lc_data[n]['phase_upsample'])) # set up phase limits if isinstance(phase_limits, str): @@ -994,14 +5196,19 @@ def plot_stack(self, title="", bin_dt=30./(60*24), dy=0.02): rprs2 = self.parameters['rprs']**2 rprs2err = 2*self.parameters['rprs']*self.errors['rprs'] - lclabel1 = r"$R^{2}_{p}/R^{2}_{s}$ = %s $\pm$ %s" %( - str(round_to_2(rprs2, rprs2err)), - str(round_to_2(rprs2err)) + rprs2_text, rprs2err_text = format_value_error_for_plot(rprs2, rprs2err) + lclabel1 = r"$(R_{p}/R_{s})^{2}$ = %s $\pm$ %s" %( + rprs2_text, + rprs2err_text, ) + tmid_text, tmid_error_text = format_value_error_for_plot( + self.parameters['tmid'], + self.errors.get('tmid',0), + ) lclabel2 = r"$T_{mid}$ = %s $\pm$ %s BJD$_{TDB}$" %( - str(round_to_2(self.parameters['tmid'], self.errors.get('tmid',0))), - str(round_to_2(self.errors.get('tmid',0))) + tmid_text, + tmid_error_text, ) lclabel = lclabel1 + "\n" + lclabel2 @@ -1016,7 +5223,7 @@ def plot_stack(self, title="", bin_dt=30./(60*24), dy=0.02): ncolor = next(colors) nmarker = next(markers) - phase = get_phase(self.lc_data[n]['time'], self.parameters['per'], self.lc_data[n]['priors']['tmid']) + phase = self.lc_data[n]['phase'] si = np.argsort(phase) bt2, br2, _ = time_bin(phase[si]*self.parameters['per'], self.lc_data[n]['residuals'][si]/np.median(self.lc_data[n]['flux'])*1e2, bin_dt) @@ -1029,17 +5236,15 @@ def plot_stack(self, title="", bin_dt=30./(60*24), dy=0.02): ax.errorbar(bt2/self.lc_data[n]['priors']['per'],bf2,yerr=bs,alpha=1,zorder=2,color=ncolor,ls='none',marker=nmarker) # replace min and max for upsampled lc model - minp = min(minp, min(phase)) - maxp = max(maxp, max(phase)) + minp = min(minp, min(self.lc_data[n]['phase_upsample'])) + maxp = max(maxp, max(self.lc_data[n]['phase_upsample'])) min_std = min(min_std, np.std(self.lc_data[n]['residuals']/np.median(self.lc_data[n]['flux']))) # best fit model - self.time_upsample = np.linspace(minp*self.parameters['per']+self.parameters['tmid'], - maxp*self.parameters['per']+self.parameters['tmid'], 10000) + self.phase_upsample = np.linspace(minp, maxp, 10000) + self.time_upsample = self.parameters['tmid'] + self.phase_upsample * self.parameters['per'] self.transit_upsample = transit(self.time_upsample, self.parameters) - self.phase_upsample = get_phase(self.time_upsample, self.parameters['per'], self.parameters['tmid']) - sii = np.argsort(self.phase_upsample) - ax.plot(self.phase_upsample[sii], self.transit_upsample[sii]-n*dy, ls='-', color=ncolor, zorder=3, label=self.lc_data[n].get('name','')) + ax.plot(self.phase_upsample, self.transit_upsample-n*dy, ls='-', color=ncolor, zorder=3, label=self.lc_data[n].get('name','')) ax.set_xlim([min(self.phase_upsample), max(self.phase_upsample)]) ax.set_xlabel("Phase ", fontsize=14) @@ -1060,8 +5265,8 @@ def plot_stack(self, title="", bin_dt=30./(60*24), dy=0.02): 'ecc': 0.5, # Eccentricity 'omega': 120, # Arg of periastron 'tmid': 0.75, # Time of mid transit [day], - 'a1': 50, # Airmass coefficients - 'a2': 0., # trend = a1 * np.exp(a2 * airmass) + 'a0': 50, # Baseline flux normalization + 'a2': 0., # trend = a0 * np.exp(a2 * (airmass - mean(airmass))) 'teff': 5000, 'tefferr': 50, @@ -1088,8 +5293,8 @@ def plot_stack(self, title="", bin_dt=30./(60*24), dy=0.02): airmass = np.zeros(time.shape[0]) # GENERATE NOISY DATA - data = transit(time, prior) * prior['a1'] * np.exp(prior['a2'] * airmass) - data += np.random.normal(0, prior['a1'] * 250e-6, len(time)) + data = transit(time, prior) * prior['a0'] * airmass_trend(prior['a2'], airmass) + data += np.random.normal(0, prior['a0'] * 250e-6, len(time)) dataerr = np.random.normal(300e-6, 50e-6, len(time)) + np.random.normal(300e-6, 50e-6, len(time)) # add bounds for free parameters only @@ -1097,9 +5302,10 @@ def plot_stack(self, title="", bin_dt=30./(60*24), dy=0.02): 'rprs': [0, 0.1], 'tmid': [prior['tmid'] - 0.01, prior['tmid'] + 0.01], 'ars': [13, 15], + # 'a0': [0.95 * prior['a0'], 1.05 * prior['a0']], # optional explicit baseline offset # 'a2': [0, 0.3] # uncomment if you want to fit for airmass - # never list 'a1' in bounds, it is perfectly correlated to exp(a2*airmass) - # and is solved for during the fit + # if a0 is omitted, the normalization is solved analytically during the fit + # never list both 'a0' and 'a1' in bounds because they are the same scale term } myfit = lc_fitter(time, data, dataerr, airmass, prior, mybounds, mode='ns') diff --git a/exotic/api/ephemeris.py b/exotic/api/ephemeris.py index d75bb476..523a96bc 100644 --- a/exotic/api/ephemeris.py +++ b/exotic/api/ephemeris.py @@ -56,6 +56,11 @@ except ImportError: from .plotting import corner +try: + from ultranest_utils import run_reactive_sampler +except ImportError: + from .ultranest_utils import run_reactive_sampler + class ephemeris_fitter(object): @@ -104,24 +109,25 @@ def fit_nested(self): def loglike(pars): # chi-squared - model = pars[0] * self.epochs + pars[1] - return -0.5 * np.sum(((self.data - model) / self.dataerr) ** 2) + data = np.asarray(self.data, dtype=float) + dataerr = np.asarray(self.dataerr, dtype=float) + pars_array = np.asarray(pars, dtype=float) + if pars_array.ndim == 2: + model = pars_array[:, 0, None] * self.epochs[None, :] + pars_array[:, 1, None] + return -0.5 * np.sum(((data[None, :] - model) / dataerr[None, :]) ** 2, axis=1) + model = pars_array[0] * self.epochs + pars_array[1] + return -0.5 * np.sum(((data - model) / dataerr) ** 2) def prior_transform(upars): # transform unit cube to prior volume return (boundarray[:, 0] + bounddiff * upars) - # estimate slope and intercept - noop = lambda *args, **kwargs: None - if self.verbose: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=4e5, - min_num_live_points=420, - show_status=True) - else: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=4e5, - min_num_live_points=420, - show_status=False, - viz_callback=noop) + sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform, vectorized=True) + self.results = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": int(4e5)}, + verbose=self.verbose, + ) # alloc data for best fit + error self.errors = {} self.quantiles = {} @@ -698,24 +704,29 @@ def fit_nested(self): def loglike(pars): # chi-squared # tmid = T0 + N*P + 0.5*dPdN*N**2 (eq 3 from paper) - model = pars[0] * self.epochs + pars[1] + 0.5 * pars[2] * self.epochs ** 2 - return -0.5 * np.sum(((self.data - model) / self.dataerr) ** 2) + data = np.asarray(self.data, dtype=float) + dataerr = np.asarray(self.dataerr, dtype=float) + pars_array = np.asarray(pars, dtype=float) + if pars_array.ndim == 2: + model = ( + pars_array[:, 0, None] * self.epochs[None, :] + + pars_array[:, 1, None] + + 0.5 * pars_array[:, 2, None] * self.epochs[None, :] ** 2 + ) + return -0.5 * np.sum(((data[None, :] - model) / dataerr[None, :]) ** 2, axis=1) + model = pars_array[0] * self.epochs + pars_array[1] + 0.5 * pars_array[2] * self.epochs ** 2 + return -0.5 * np.sum(((data - model) / dataerr) ** 2) def prior_transform(upars): # transform unit cube to prior volume return (boundarray[:, 0] + bounddiff * upars) - # estimate slope and intercept - noop = lambda *args, **kwargs: None - if self.verbose: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=4e5, - min_num_live_points=420, - show_status=True) - else: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=4e5, - min_num_live_points=420, - show_status=False, - viz_callback=noop) + sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform, vectorized=True) + self.results = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": int(4e5)}, + verbose=self.verbose, + ) # alloc data for best fit + error self.errors = {} self.quantiles = {} @@ -1146,4 +1157,4 @@ def plot_triangle(self): for key in nlf.parameters: print(f"Parameter {key} = {nlf.parameters[key]:.2e} +- {nlf.errors[key]:.2e}") - # TODO BIC Values \ No newline at end of file + # TODO BIC Values diff --git a/exotic/api/filters.py b/exotic/api/filters.py index eaeceb75..14ec7e6b 100644 --- a/exotic/api/filters.py +++ b/exotic/api/filters.py @@ -9,6 +9,11 @@ "Johnson R": {"name": "RJ", "fwhm": ("590.0", "810.0")}, "Johnson I": {"name": "IJ", "fwhm": ("780.0", "1020.0")}, + # Photographic + "Photographic B": {"name": "PB", "fwhm": ("391.6", "480.6")}, + "Photographic G": {"name": "PG", "fwhm": ("502.8", "586.8")}, + "Photographic R": {"name": "PR", "fwhm": ("590.0", "810.0")}, + # Cousins "Cousins R": {"name": "R", "fwhm": ("561.7", "719.7")}, "Cousins I": {"name": "I", "fwhm": ("721.0", "875.0")}, @@ -43,11 +48,12 @@ "PanSTARRS Y": {"name": "Y", "fwhm": ("946.4", "1054.4")}, "PanSTARRS w": {"name": "N/A", "fwhm": ("404.2", "845.8")}, - # MObs Clear Filter; Source(s): Martin Fowler - "MObs CV": {"name": "CV", "fwhm": ("350.0", "850.0")}, + # ClearV filter + "CV": {"name": "CV", "fwhm": ("350.0", "1000.0")}, - # Astrodon CBB; Source(s): George Silvis; https://astrodon.com/products/astrodon-exo-planet-filter/ - "Astrodon ExoPlanet-BB": {"name": "CBB", "fwhm": ("500.0", "1000.0")}, + # Clear with blue-blocking (CBB); wavelength source: + # https://astrodon.com/products/astrodon-exo-planet-filter/ + "CBB": {"name": "CBB", "fwhm": ("500.0", "1000.0")}, } # expose as fwhm and for convenience set 'desc' field equal to key fwhm = {k: v for k, v in __fwhm.items() if (v.update(desc=k),)} @@ -72,18 +78,59 @@ "LCO Pan-STARRS Y": "PanSTARRS Y", "LCO Pan-STARRS w": "PanSTARRS w", - "Clear (unfiltered) reduced to V sequence": "MObs CV", + "Clear (unfiltered) reduced to V sequence": "CV", "Clear (unfiltered) reduced to R sequence": "Cousins R", - "Clear with blue-blocking": "Astrodon ExoPlanet-BB", - "Exop": "Astrodon ExoPlanet-BB", + "Clear with blue-blocking": "CBB", + "Astrodon ExoPlanet-BB": "CBB", + "Astrodon-Exo": "CBB", + "Exop": "CBB", + + # additional short aliases found in FILTER column values + "bu": "Johnson U", + "bb": "Johnson B", + "pb": "Photographic B", + "bv": "Johnson V", + "G": "Photographic G", + "pg": "Photographic G", + "br": "Johnson R", + "pr": "Photographic R", + "bi": "Johnson I", + "up": "Sloan u", + "gp": "Sloan g", + "rp": "Sloan r", + "ip": "Sloan i", + "zp": "Sloan z", + "su": "Stromgren u", + "sv": "Stromgren v", + "sb": "Stromgren b", + "sy": "Stromgren y", + "hb": "Stromgren Hbw", + "zs": "PanSTARRS z-short", + "MObs CV": "CV", + "ClearV": "CV", + "clearV": "CV", + "C": "CV", + "clear": "CV", + "lum": "CV", + "Lum": "CV", + "Luminosity": "CV", + "luminosity": "CV", + "w": "CV", + "pl": "CV", + "exo": "CBB", + + # OSC split-channel aliases + "b1": "Photographic B", + "g1": "Photographic G", + "g2": "Photographic G", + "r1": "Photographic R", + "r2": "Photographic R", } # standard filters w/o precisely defined FWHM values fwhm_names_nonspecific = { 'CR': "Clear (unfiltered) reduced to R sequence", - 'CBB': "Clear with blue-blocking", - 'CV': "Clear (unfiltered) reduced to V sequence", 'TB': "DSLR Blue", 'TG': "DSLR Green", 'TR': "DSLR Red", diff --git a/exotic/api/gael_ld.py b/exotic/api/gael_ld.py index 1b42aa10..4f253a5e 100644 --- a/exotic/api/gael_ld.py +++ b/exotic/api/gael_ld.py @@ -41,9 +41,26 @@ import logging import matplotlib.pyplot as plt import numpy as np +import os +from html.parser import HTMLParser +from pathlib import Path +from urllib.parse import quote +from urllib.parse import unquote + +import requests log = logging.getLogger(__name__) +_LDTK_HTTP_FALLBACK_BASE_URLS = ( + "https://ftp.gwdg.de/pub/misc/phoenix", + "https://downloads.nextastro.org/PHOENIX", +) +_LDTK_HTTP_FALLBACK_ENV = "EXOTIC_LDTK_FALLBACK_BASE_URL" +_LDTK_DOWNLOAD_TIMEOUT = (10, 120) +_LDTK_INDEX_TIMEOUT = (10, 60) +_LDTK_ORIGINAL_GET_SERVER_FILE_LIST = None +_LDTK_ORIGINAL_DOWNLOAD_UNCACHED_FILES = None + class LDPSet(ldtk.LDPSet): """ @@ -61,6 +78,235 @@ def profile_mu(self): return self._mu setattr(ldtk.ldtk, 'LDPSet', LDPSet) +def _ldtk_http_fallback_base_urls(): + configured_urls = os.environ.get(_LDTK_HTTP_FALLBACK_ENV) + if configured_urls is None: + return [url.rstrip("/") for url in _LDTK_HTTP_FALLBACK_BASE_URLS] + return [url.strip().rstrip("/") for url in configured_urls.split(",") if url.strip()] + + +def _quote_url_path(*parts): + segments = [] + for part in parts: + segments.extend(segment for segment in str(part).strip("/").split("/") if segment) + return "/".join(quote(segment, safe="") for segment in segments) + + +def _ldtk_http_fallback_url(base_url, client, ldtk_file): + path = _quote_url_path(client.edir, ldtk_file._zstr, ldtk_file.name) + return f"{base_url}/{path}" + + +def _ldtk_http_fallback_index_url(base_url, *parts): + path = _quote_url_path(*parts) + if path: + return f"{base_url}/{path}/" + return f"{base_url}/" + + +class _HrefParser(HTMLParser): + def __init__(self): + super().__init__() + self.hrefs = [] + + def handle_starttag(self, tag, attrs): + if tag.lower() != "a": + return + for name, value in attrs: + if name.lower() == "href" and value: + self.hrefs.append(value) + return + + +def _http_index_names(url): + response = requests.get(url, timeout=_LDTK_INDEX_TIMEOUT) + try: + response.raise_for_status() + parser = _HrefParser() + parser.feed(response.text) + finally: + response.close() + + names = [] + for href in parser.hrefs: + href = unquote(href.split("?", 1)[0].split("#", 1)[0]).strip("/") + if not href or href in (".", "..") or "/" in href: + continue + names.append(href) + return names + + +def _get_ldtk_server_file_list_from_http_mirror(client, base_url): + root_url = _ldtk_http_fallback_index_url(base_url, client.edir) + zdirs = sorted(name for name in _http_index_names(root_url) if ".txt" not in name.lower()) + if not zdirs: + raise RuntimeError(f"No PHOENIX metallicity directories found at {root_url}") + + log.warning( + "Trying PHOENIX HTTP fallback index at %s for %d metallicity directories.", + base_url, + len(zdirs), + ) + + files_in_server = {} + for zdir in zdirs: + zdir_url = _ldtk_http_fallback_index_url(base_url, client.edir, zdir) + files_in_server[zdir] = sorted( + name for name in _http_index_names(zdir_url) if ".txt" not in name.lower() + ) + return files_in_server + + +def _get_ldtk_server_file_list_from_http(client): + base_urls = _ldtk_http_fallback_base_urls() + if not base_urls: + raise RuntimeError( + f"LDTk FTP file listing failed and {_LDTK_HTTP_FALLBACK_ENV} is empty, " + "so EXOTIC cannot try the HTTP PHOENIX fallback." + ) + + log.warning( + "LDTk FTP file listing failed; trying PHOENIX HTTP fallback mirrors in order: %s", + ", ".join(base_urls), + ) + + last_error = None + for base_url in base_urls: + try: + return _get_ldtk_server_file_list_from_http_mirror(client, base_url) + except Exception as mirror_error: + last_error = mirror_error + log.warning("PHOENIX HTTP fallback index failed at %s: %s", base_url, mirror_error) + + raise RuntimeError("All PHOENIX HTTP fallback indexes failed.") from last_error + + +def _download_file(url, local_path): + local_path = Path(local_path) + local_path.parent.mkdir(parents=True, exist_ok=True) + temporary_path = local_path.with_name(f"{local_path.name}.download") + try: + response = requests.get(url, stream=True, timeout=_LDTK_DOWNLOAD_TIMEOUT) + try: + response.raise_for_status() + with open(temporary_path, "wb") as local_file: + for chunk in response.iter_content(chunk_size=1024 * 1024): + if chunk: + local_file.write(chunk) + finally: + response.close() + os.replace(temporary_path, local_path) + except Exception: + if temporary_path.exists(): + temporary_path.unlink() + raise + + +def _ldtk_files_to_download(client, force=False): + return [ldtk_file for ldtk_file in client.files if force or not ldtk_file.local_exists] + + +def _download_ldtk_uncached_files_from_http_mirror(client, base_url, force=False): + files_to_download = _ldtk_files_to_download(client, force=force) + if not files_to_download: + return False + + log.warning( + "Trying PHOENIX HTTP fallback at %s for %d file(s).", + base_url, + len(files_to_download), + ) + + downloaded_paths = [] + for ldtk_file in files_to_download: + url = _ldtk_http_fallback_url(base_url, client, ldtk_file) + _download_file(url, ldtk_file.local_path) + downloaded_paths.append(ldtk_file.local_path) + if client.not_cached > 0 and not force: + client.not_cached -= 1 + + if client.check_file_corruption(downloaded_paths): + raise RuntimeError("Downloaded PHOENIX files failed LDTk's FITS corruption check.") + return False + + +def _download_ldtk_uncached_files_from_http(client, force=False): + base_urls = _ldtk_http_fallback_base_urls() + if not base_urls: + raise RuntimeError( + f"LDTk FTP download failed and {_LDTK_HTTP_FALLBACK_ENV} is empty, " + "so EXOTIC cannot try the HTTP PHOENIX fallback." + ) + + log.warning( + "LDTk FTP download failed; trying PHOENIX HTTP fallback mirrors in order: %s", + ", ".join(base_urls), + ) + + last_error = None + for base_url in base_urls: + try: + return _download_ldtk_uncached_files_from_http_mirror(client, base_url, force=force) + except Exception as mirror_error: + last_error = mirror_error + log.warning("PHOENIX HTTP fallback mirror failed at %s: %s", base_url, mirror_error) + + raise RuntimeError("All PHOENIX HTTP fallback mirrors failed.") from last_error + + +def _install_ldtk_http_fallback(): + global _LDTK_ORIGINAL_GET_SERVER_FILE_LIST + global _LDTK_ORIGINAL_DOWNLOAD_UNCACHED_FILES + + try: + from ldtk.client import Client + except Exception: + return + + if not getattr(Client.get_server_file_list, "_exotic_http_fallback", False): + _LDTK_ORIGINAL_GET_SERVER_FILE_LIST = Client.get_server_file_list + + def get_server_file_list_with_http_fallback(self): + try: + return _LDTK_ORIGINAL_GET_SERVER_FILE_LIST(self) + except Exception as ftp_error: + try: + log.warning("LDTk FTP file listing failed with %s", ftp_error) + return _get_ldtk_server_file_list_from_http(self) + except Exception as fallback_error: + raise RuntimeError( + "LDTk could not list PHOENIX files from the default FTP server " + "or the EXOTIC HTTP fallback." + ) from fallback_error + + get_server_file_list_with_http_fallback._exotic_http_fallback = True + Client.get_server_file_list = get_server_file_list_with_http_fallback + + if getattr(Client.download_uncached_files, "_exotic_http_fallback", False): + return + + _LDTK_ORIGINAL_DOWNLOAD_UNCACHED_FILES = Client.download_uncached_files + + def download_uncached_files_with_http_fallback(self, force=False): + try: + return _LDTK_ORIGINAL_DOWNLOAD_UNCACHED_FILES(self, force=force) + except Exception as ftp_error: + try: + log.warning("LDTk FTP download failed with %s", ftp_error) + return _download_ldtk_uncached_files_from_http(self, force=force) + except Exception as fallback_error: + raise RuntimeError( + "LDTk could not download PHOENIX files from the default FTP server " + "or the EXOTIC HTTP fallback." + ) from fallback_error + + download_uncached_files_with_http_fallback._exotic_http_fallback = True + Client.download_uncached_files = download_uncached_files_with_http_fallback + + +_install_ldtk_http_fallback() + + def createldgrid(minmu, maxmu, orbp, ldmodel='nonlinear', phoenixmin=1e-1, segmentation=int(10), verbose=False): @@ -129,7 +375,11 @@ def createldgrid(minmu, maxmu, orbp, out['LD'] = allcl.T out['ERR'] = allel.T for i in range(0, len(allcl.T)): - log.warning(f">-- LD{int(i)}: {float(allcl.T[i])} +/- {float(allel.T[i])}") + ld_value = np.ravel(np.asarray(allcl.T[i], dtype=float)) + err_value = np.ravel(np.asarray(allel.T[i], dtype=float)) + if ld_value.size == 0 or err_value.size == 0: + continue + log.warning(f">-- LD{int(i)}: {float(ld_value[0])} +/- {float(err_value[0])}") pass return out diff --git a/exotic/api/http_compression.py b/exotic/api/http_compression.py new file mode 100644 index 00000000..3ee06009 --- /dev/null +++ b/exotic/api/http_compression.py @@ -0,0 +1,40 @@ +import gzip +import json + +try: + import zstandard +except ImportError: # pragma: no cover - gzip fallback covers environments without zstandard + zstandard = None + + +_GZIP_LEVEL = 6 +_ZSTD_LEVEL = 3 + + +def build_compressed_json_request(payload, content_encoding=None): + raw_body = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + normalized_encoding = None if content_encoding is None else str(content_encoding).strip().lower() + if normalized_encoding not in (None, "", "gzip", "zstd"): + raise ValueError(f"Unsupported content encoding: {content_encoding}") + + if normalized_encoding == "zstd": + if zstandard is None: + raise RuntimeError("zstandard compression requested, but the zstandard package is unavailable.") + compressed_body = zstandard.ZstdCompressor(level=_ZSTD_LEVEL).compress(raw_body) + encoding = "zstd" + elif normalized_encoding == "gzip": + compressed_body = gzip.compress(raw_body, compresslevel=_GZIP_LEVEL) + encoding = "gzip" + elif zstandard is not None: + compressed_body = zstandard.ZstdCompressor(level=_ZSTD_LEVEL).compress(raw_body) + encoding = "zstd" + else: + compressed_body = gzip.compress(raw_body, compresslevel=_GZIP_LEVEL) + encoding = "gzip" + + headers = { + "Content-Encoding": encoding, + "Content-Type": "application/json", + } + return compressed_body, headers, encoding, len(raw_body), len(compressed_body) diff --git a/exotic/api/joint_fitter.py b/exotic/api/joint_fitter.py index 035be933..8257f009 100644 --- a/exotic/api/joint_fitter.py +++ b/exotic/api/joint_fitter.py @@ -38,24 +38,64 @@ from astropy import constants as const from astropy import units as u from copy import deepcopy +from contextlib import redirect_stderr, redirect_stdout +import faulthandler +import io from itertools import cycle +import os +import sys import matplotlib.pyplot as plt import numpy as np -from pylightcurve.models.exoplanet_lc import eclipse_mid_time, transit_flux_drop from scipy import stats -try: - from ultranest import ReactiveNestedSampler -except ImportError: - import dynesty - import dynesty.plotting - from dynesty.utils import resample_equal - from scipy.stats import gaussian_kde +from ultranest import ReactiveNestedSampler try: from elca import glc_fitter, lc_fitter except ImportError: from .elca import glc_fitter, lc_fitter +try: + from ultranest_utils import run_reactive_sampler +except ImportError: + from .ultranest_utils import run_reactive_sampler + +def _pylightcurve_import_watchdog_seconds(): + try: + return float(os.environ.get("EXOTIC_IMPORT_WATCHDOG_SECONDS", "120")) + except (TypeError, ValueError): + return 120.0 + + +def _start_import_watchdog(): + timeout = _pylightcurve_import_watchdog_seconds() + if timeout <= 0: + return False + + try: + if not faulthandler.is_enabled(): + faulthandler.enable(file=sys.__stdout__, all_threads=True) + faulthandler.dump_traceback_later(timeout, repeat=True, file=sys.__stdout__) + return True + except Exception: + return False + + +def _load_pylightcurve_symbols(): + watchdog_started = _start_import_watchdog() + try: + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + from pylightcurve.models.exoplanet_lc import eclipse_mid_time, transit + return eclipse_mid_time, transit + finally: + if watchdog_started: + try: + faulthandler.cancel_dump_traceback_later() + except Exception: + pass + + +eclipse_mid_time, _pylightcurve_transit = _load_pylightcurve_symbols() + AU = const.au.to(u.m).value Mjup = const.M_jup.to(u.kg).value Msun = const.M_sun.to(u.kg).value @@ -111,15 +151,8 @@ def planet_orbit(period, sma_over_rs, eccentricity, inclination, periastron, mid def pytransit(limb_darkening_coefficients, rp_over_rs, period, sma_over_rs, eccentricity, inclination, periastron, mid_time, time_array, method='claret', precision=3): - - position_vector = planet_orbit(period, sma_over_rs, eccentricity, inclination, periastron, mid_time, time_array) - - projected_distance = np.where( - position_vector[0] < 0, 1.0 + 5.0 * rp_over_rs, - np.sqrt(position_vector[1] * position_vector[1] + position_vector[2] * position_vector[2])) - - return transit_flux_drop(limb_darkening_coefficients, rp_over_rs, projected_distance, - method=method, precision=precision) + return _pylightcurve_transit(limb_darkening_coefficients, rp_over_rs, period, sma_over_rs, eccentricity, + inclination, periastron, mid_time, time_array, method=method, precision=precision) def transit(times, values): model = pytransit([values['u0'], values['u1'], values['u2'], values['u3']], @@ -128,9 +161,6 @@ def transit(times, values): values['tmid'], times, method='claret', precision=3) return model -from pylightcurve.models.exoplanet_lc import transit as pytransit -from pylightcurve.models.exoplanet_lc import eclipse_mid_time - def eclipse(times, values): tme = eclipse_mid_time(values['per'], values['ars'], values['ecc'], values['inc'], values['omega'], values['tmid']) model = pytransit([0,0,0,0], @@ -421,10 +451,12 @@ def loglike(pars): for k in lfreekeys[n]: freekeys.append(f"local_{n}_{k}") - if self.verbose: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=2e5) - else: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=2e5, show_status=self.verbose, viz_callback=self.verbose) + sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform) + self.results = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": int(2e5)}, + verbose=self.verbose, + ) try: self.parameters = deepcopy(self.lc_data[0]['priors']) @@ -886,4 +918,4 @@ def plot_oc_eclipses(self): ax.set_ylabel("Residuals [min]",fontsize=14) ax.grid(True, ls='--') plt.tight_layout() - return fig \ No newline at end of file + return fig diff --git a/exotic/api/ld.py b/exotic/api/ld.py index 5e939f1e..ccaa6c39 100644 --- a/exotic/api/ld.py +++ b/exotic/api/ld.py @@ -65,7 +65,8 @@ class LimbDarkening: # lookup table: fwhm_lookup references filters irrespective of spacing and punctuation # 1 - combine optimized str lookups in lookup table fwhm_lookup = {k.strip().replace(' ', '').lower(): k for k in fwhm.keys()} - fwhm_lookup.update({k.strip().replace(' ', '').lower(): v for k, v in fwhm_alias.items()}) + for k, v in fwhm_alias.items(): + fwhm_lookup.setdefault(k.strip().replace(' ', '').lower(), v) # 2 - ignore punctuation in lookup table fwhm_lookup = {re.sub(ld_re_punct_p, '', k): v for k, v in fwhm_lookup.items()} # lookup set: filter_desc_nonspecific_lookup_set references descriptions that do not represent a specific filter @@ -143,13 +144,20 @@ def check_standard(self, filter_: dict = None, loose: bool = False, loose_len: i if k == 'name' and filter_[k]: # format 'name' (if exists) to uppercase, no spaces filter_[k] = filter_[k].upper().replace(' ', '') if filter_['filter']: # make matcher by removing spaces, remove punctuation and lowercase + if filter_['filter'] in LimbDarkening.fwhm_alias: + filter_['filter'] = LimbDarkening.fwhm_alias[filter_['filter']] filter_matcher = filter_['filter'].lower().replace(' ', '') filter_matcher = re.sub(ld_re_punct_p, '', filter_matcher) # names that do not represent a specific filter combined into one tuple filter_names_nonspecific = set(LimbDarkening.fwhm_names_nonspecific.keys()) filter_names_nonspecific.update(LimbDarkening.filter_names_undefined) + # prefer explicit all-uppercase abbreviations (e.g. 'SU') before loose lookup aliases + if (filter_['filter'] and filter_['filter'] == filter_['filter'].upper() and + filter_['filter'].strip() not in filter_names_nonspecific): + filter_alias = next((f for f in LimbDarkening.fwhm.values() + if filter_['filter'].strip() == f['name'].strip().upper()), None) # identify defined filters via optimized lookup table - if (filter_matcher and filter_matcher in LimbDarkening.fwhm_lookup and + if (not filter_alias and filter_matcher and filter_matcher in LimbDarkening.fwhm_lookup and filter_matcher not in LimbDarkening.filter_desc_nonspecific_lookup_set): filter_['filter'] = LimbDarkening.fwhm_lookup[filter_matcher] # sets to actual filter reference key for f in LimbDarkening.fwhm.values(): @@ -216,7 +224,14 @@ def check_fwhm(filter_: dict = None) -> bool: return False for k in ('wl_min', 'wl_max'): # clean inputs filter_[k] = filter_.get(k) - filter_[k] = str(filter_[k]).strip().replace(' ', '').rstrip('.') if filter_[k] else filter_[k] + if filter_[k] is None: + continue + filter_[k] = str(filter_[k]).strip().replace(' ', '').rstrip('.') + if not filter_[k]: + filter_[k] = None + if filter_['wl_min'] is None or filter_['wl_max'] is None: + return False + for k in ('wl_min', 'wl_max'): if not 200. <= float(filter_[k]) <= 4000.: # also fails if nan raise ValueError(f"FWHM '{k}' is outside of bounds (200., 4000.). ...") else: # add .0 to end of str to aid literal matching diff --git a/exotic/api/nbody.py b/exotic/api/nbody.py index 068cf0cd..e4d0784a 100644 --- a/exotic/api/nbody.py +++ b/exotic/api/nbody.py @@ -46,7 +46,7 @@ import numpy as np import matplotlib.pyplot as plt import rebound -from exotic.api.plotting import corner +from exotic.api.ultranest_utils import run_reactive_sampler from ultranest import ReactiveNestedSampler from astropy.io import fits from astropy import units as u @@ -598,11 +598,12 @@ def loglike(pars): def prior_transform(upars): return (boundarray[:,0] + bounddiff*upars) - if self.verbose: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=1e5) - else: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=1e5, show_status=self.verbose, -viz_callback=self.verbose) + sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform) + self.results = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": int(1e5)}, + verbose=self.verbose, + ) self.errors = {} self.quantiles = {} @@ -752,4 +753,4 @@ def prior_transform(upars): nfit = nbody_fitter(data, nbody_prior, nbody_bounds) # print(nfit.parameters) - # print(nfit.errors) \ No newline at end of file + # print(nfit.errors) diff --git a/exotic/api/nea.py b/exotic/api/nea.py index dbafcf1b..a48ae20c 100644 --- a/exotic/api/nea.py +++ b/exotic/api/nea.py @@ -47,7 +47,7 @@ import requests import time import urllib.parse -from tenacity import retry, retry_if_exception_type, stop_after_attempt, \ +from tenacity import RetryError, retry, retry_if_exception_type, stop_after_attempt, \ wait_exponential # constants @@ -64,12 +64,54 @@ def result_if_max_retry_count(retry_state): pass +def _strip_observation_phase_suffix(name): + """Remove scheduler phase labels that are not part of a target name.""" + text = str(name or '').strip() + return re.sub(r'(?:\s+(?:ingress|egress))+\s*$', '', text, flags=re.IGNORECASE).strip() + + +def _collapse_number_planet_letter_spaces(name): + """Keep a trailing single planet letter attached to its numeric identifier.""" + return re.sub(r'(?<=\d)\s+(?=[a-z](?:\s|$))', '', str(name or '').strip()) + + +def planet_name_lookup_candidates(name): + """Return progressively smaller names for tolerant archive matching. + + The exact value is retained first. Observation-phase suffixes are then + removed, spaces between a number and a single planet letter are collapsed, + and finally each contiguous group of remaining space-separated terms is + offered from longest to shortest. + """ + candidates = [] + + def add(value): + value = str(value or '').strip() + if value and value not in candidates: + candidates.append(value) + + original = str(name or '').strip() + add(original) + without_phase = _strip_observation_phase_suffix(original) + add(without_phase) + collapsed = _collapse_number_planet_letter_spaces(without_phase) + add(collapsed) + + parts = collapsed.split() + for width in range(len(parts) - 1, 0, -1): + for start in range(0, len(parts) - width + 1): + add(' '.join(parts[start:start + width])) + + return candidates + + class NASAExoplanetArchive: - def __init__(self, planet=None, candidate=False): + def __init__(self, planet=None, candidate=False, non_interactive=False): self.planet = planet # self.candidate = candidate self.pl_dict = None + self.non_interactive = bool(non_interactive) # CONFIGURATIONS self.requests_timeout = 16, 512 # connection timeout, response timeout in secs. @@ -114,7 +156,14 @@ def planet_info(self, fancy=False): return json.dumps(flabels, indent=4) else: - self.planet, candidate = self._new_scrape(filename="eaConf.json") + try: + self.planet, candidate = self._new_scrape(filename="eaConf.json") + except (RetryError, requests.exceptions.RequestException, ConnectionError): + if not self._load_params_from_nextastro_cache(): + raise + candidate = False + print(f"Successfully found {self.planet} in NextAstro cached NASA Exoplanet Archive parameters!") + return self.planet, candidate, self.pl_dict if not candidate: with open("eaConf.json", "r") as confirmed: @@ -126,6 +175,105 @@ def planet_info(self, fancy=False): return self.planet, candidate, self.pl_dict + @staticmethod + def _extract_value_and_errors(payload): + if not isinstance(payload, dict): + return payload, None, None + + value = payload.get('value') + err_plus = payload.get('errPlus') + err_minus = payload.get('errMinus') + return value, err_plus, err_minus + + @staticmethod + def _negative_error(value): + if value is None: + return None + return -abs(value) + + @staticmethod + def _candidate_name_reason(name): + if not isinstance(name, str): + return None + + normalized = name.strip().upper() + if not normalized: + return None + + if normalized.startswith('TIC'): + return "the name starts with 'TIC'" + + if re.search(r'\.\d{2,}$', normalized): + return "the name ends with a decimal suffix" + + return None + + def _load_params_from_nextastro_cache(self): + if not self.planet: + return False + + endpoint = "https://archive.nextastro.org/api/exoplanet_params" + response = requests.get( + endpoint, + params={'name': self.planet}, + timeout=self.requests_timeout + ) + response.raise_for_status() + payload = response.json() + params = payload.get('params') if isinstance(payload, dict) else None + + if not isinstance(params, dict): + return False + + period, period_ep, period_em = self._extract_value_and_errors(params.get('orbitalPeriodDays')) + midt, midt_ep, midt_em = self._extract_value_and_errors(params.get('midTransitTimeDays')) + rprs, rprs_ep, rprs_em = self._extract_value_and_errors(params.get('rpOverRs')) + ars, ars_ep, ars_em = self._extract_value_and_errors(params.get('aOverRs')) + incl, incl_ep, incl_em = self._extract_value_and_errors(params.get('inclinationDeg')) + teff, teff_ep, teff_em = self._extract_value_and_errors(params.get('starTeffK')) + feh, feh_ep, feh_em = self._extract_value_and_errors(params.get('starFeh')) + logg, logg_ep, logg_em = self._extract_value_and_errors(params.get('starLogg')) + + mapped_data = { + 'pl_name': params.get('name', self.planet), + 'hostname': params.get('hostStarName'), + 'ra': params.get('raDeg'), + 'dec': params.get('decDeg'), + 'pl_orbper': period, + 'pl_orbpererr1': period_ep, + 'pl_orbpererr2': self._negative_error(period_em), + 'pl_tranmid': midt, + 'pl_tranmiderr1': midt_ep, + 'pl_tranmiderr2': self._negative_error(midt_em), + 'pl_ratror': rprs, + 'pl_ratrorerr1': rprs_ep, + 'pl_ratrorerr2': self._negative_error(rprs_em), + 'pl_ratdor': ars, + 'pl_ratdorerr1': ars_ep, + 'pl_ratdorerr2': self._negative_error(ars_em), + 'pl_orbincl': incl, + 'pl_orbinclerr1': incl_ep, + 'pl_orbinclerr2': self._negative_error(incl_em), + 'pl_orbeccen': params.get('eccentricity'), + 'pl_orblper': params.get('argPeriastronDeg'), + 'st_teff': teff, + 'st_tefferr1': teff_ep, + 'st_tefferr2': self._negative_error(teff_em), + 'st_met': feh, + 'st_meterr1': feh_ep, + 'st_meterr2': self._negative_error(feh_em), + 'st_logg': logg, + 'st_loggerr1': logg_ep, + 'st_loggerr2': self._negative_error(logg_em), + 'sy_dist': None, + 'sy_pmra': None, + 'sy_pmdec': None, + } + + self.planet = mapped_data['pl_name'] + self._get_params(mapped_data) + return True + @staticmethod def dataframe_to_jsonfile(dataframe, filename): jsondata = json.loads(dataframe.to_json(orient='table', index=False)) @@ -224,12 +372,12 @@ def _new_scrape(self, filename="eaConf.json"): if os.path.exists('pl_names.json'): with open("pl_names.json", "r") as f: planets = json.load(f) - planet_key = re.sub(r'[^a-zA-Z0-9]', '', self.planet.lower()) - - planet_exists = planets.get(planet_key, False) - - if planet_exists: - self.planet = planet_exists + for candidate_name in planet_name_lookup_candidates(self.planet): + planet_key = re.sub(r'[^a-zA-Z0-9]', '', candidate_name.lower()) + planet_exists = planets.get(planet_key, False) + if planet_exists: + self.planet = planet_exists + break print(f"\nLooking up {self.planet} on the NASA Exoplanet Archive. Please wait....") @@ -247,6 +395,19 @@ def _new_scrape(self, filename="eaConf.json"): extra = self._tap_query(uri_ipac_base, uri_ipac_query) if len(default) == 0: + candidate_reason = self._candidate_name_reason(self.planet) + if candidate_reason: + print(f"Cannot find target ({self.planet}) in NASA Exoplanet Archive." + f"\nAssuming {self.planet} is a planet candidate because {candidate_reason}.") + return self.planet, True + + if self.non_interactive: + raise RuntimeError( + f"Non-interactive run cancelled: target ({self.planet}) was not found in the NASA " + "Exoplanet Archive, so archive coordinates are unavailable. Check the Planet Name " + "in the initialization file or provide valid target RA and Dec coordinates." + ) + self.planet = input(f"Cannot find target ({self.planet}) in NASA Exoplanet Archive." f"\nPlease go to https://exoplanetarchive.ipac.caltech.edu to check naming and" "\nre-enter the planet's name or type 'candidate' if this is a planet candidate: ") diff --git a/exotic/api/nested_linear_fitter.py b/exotic/api/nested_linear_fitter.py index 677d8b1a..81baecaf 100644 --- a/exotic/api/nested_linear_fitter.py +++ b/exotic/api/nested_linear_fitter.py @@ -55,6 +55,11 @@ except ImportError: from .plotting import corner +try: + from ultranest_utils import run_reactive_sampler +except ImportError: + from .ultranest_utils import run_reactive_sampler + class linear_fitter(object): def __init__(self, data, dataerr, bounds=None, prior=None, labels=None, verbose=True): @@ -97,24 +102,25 @@ def fit_nested(self): def loglike(pars): # chi-squared - model = pars[0] * self.epochs + pars[1] - return -0.5 * np.sum(((self.data - model) / self.dataerr) ** 2) + pars_array = np.asarray(pars, dtype=float) + data = np.asarray(self.data, dtype=float) + dataerr = np.asarray(self.dataerr, dtype=float) + if pars_array.ndim == 2: + model = pars_array[:, 0, None] * self.epochs[None, :] + pars_array[:, 1, None] + return -0.5 * np.sum(((data[None, :] - model) / dataerr[None, :]) ** 2, axis=1) + model = pars_array[0] * self.epochs + pars_array[1] + return -0.5 * np.sum(((data - model) / dataerr) ** 2) def prior_transform(upars): # transform unit cube to prior volume return (boundarray[:, 0] + bounddiff * upars) - # estimate slope and intercept - noop = lambda *args, **kwargs: None - if self.verbose: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=4e5, - min_num_live_points=420, - show_status=True) - else: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=4e5, - min_num_live_points=420, - show_status=False, - viz_callback=noop) + sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform, vectorized=True) + self.results = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": int(4e5)}, + verbose=self.verbose, + ) # alloc data for best fit + error self.errors = {} self.quantiles = {} @@ -670,24 +676,29 @@ def fit_nested(self): def loglike(pars): # chi-squared # tmid = t0 + N*P + 0.5*dPdN*N**2 (eq 3 from paper) - model = pars[0] * self.epochs + pars[1] + 0.5 * pars[2] * self.epochs ** 2 - return -0.5 * np.sum(((self.data - model) / self.dataerr) ** 2) + pars_array = np.asarray(pars, dtype=float) + data = np.asarray(self.data, dtype=float) + dataerr = np.asarray(self.dataerr, dtype=float) + if pars_array.ndim == 2: + model = ( + pars_array[:, 0, None] * self.epochs[None, :] + + pars_array[:, 1, None] + + 0.5 * pars_array[:, 2, None] * self.epochs[None, :] ** 2 + ) + return -0.5 * np.sum(((data[None, :] - model) / dataerr[None, :]) ** 2, axis=1) + model = pars_array[0] * self.epochs + pars_array[1] + 0.5 * pars_array[2] * self.epochs ** 2 + return -0.5 * np.sum(((data - model) / dataerr) ** 2) def prior_transform(upars): # transform unit cube to prior volume return (boundarray[:, 0] + bounddiff * upars) - # estimate slope and intercept - noop = lambda *args, **kwargs: None - if self.verbose: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=4e5, - min_num_live_points=420, - show_status=True) - else: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=4e5, - min_num_live_points=420, - show_status=False, - viz_callback=noop) + sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform, vectorized=True) + self.results = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": int(4e5)}, + verbose=self.verbose, + ) # alloc data for best fit + error self.errors = {} self.quantiles = {} diff --git a/exotic/api/output_aavso.py b/exotic/api/output_aavso.py index 9e237056..9a8081b5 100644 --- a/exotic/api/output_aavso.py +++ b/exotic/api/output_aavso.py @@ -37,20 +37,121 @@ # ########################################################################### # import hashlib from json import dump, dumps +import math from numpy import mean, median, std -import os from pathlib import Path import re -from tkinter import NONE try: - from .utils import round_to_2 + from ..utils import ( + aavso_output_directory, + format_aavso_exoplanet_name, + format_value_and_uncertainty, + format_value_with_uncertainty, + round_to_2, + safe_output_filename, + ) except ImportError: - from utils import round_to_2 + from utils import ( + aavso_output_directory, + format_aavso_exoplanet_name, + format_value_and_uncertainty, + format_value_with_uncertainty, + round_to_2, + safe_output_filename, + ) try: from .version import __version__ except ImportError: from version import __version__ +try: + from ..transit_depth import ( + AREA_DEPTH_LABEL, + OBSERVABLE_DEPTH_DELTA_LABEL, + OBSERVABLE_DEPTH_LABEL, + PRIOR_OBSERVABLE_DEPTH_LABEL, + fit_transit_depth_summary, + planet_dict_transit_errors, + planet_dict_transit_parameters, + ) +except ImportError: + from exotic.transit_depth import ( + AREA_DEPTH_LABEL, + OBSERVABLE_DEPTH_DELTA_LABEL, + OBSERVABLE_DEPTH_LABEL, + PRIOR_OBSERVABLE_DEPTH_LABEL, + fit_transit_depth_summary, + planet_dict_transit_errors, + planet_dict_transit_parameters, + ) + + +def _format_depth(value, error): + try: + value = float(value) + except (TypeError, ValueError): + return None + if not math.isfinite(value): + return None + try: + error = float(error) + except (TypeError, ValueError): + error = math.nan + if math.isfinite(error) and error >= 0: + return f"{format_value_with_uncertainty(value, error)} [%]" + return f"{round_to_2(value)} +/- n/a [%]" + + +def _depth_final_params(fit, planet_dict=None, limb_darkening=None): + depth_summary = fit_transit_depth_summary( + fit, + prior_parameters=planet_dict_transit_parameters( + planet_dict, + limb_darkening=limb_darkening, + fallback=getattr(fit, 'prior', None), + ), + prior_errors=planet_dict_transit_errors(planet_dict, limb_darkening=limb_darkening), + ) + entries = {} + for label, value_key, error_key in ( + (AREA_DEPTH_LABEL, 'area_depth', 'area_depth_error'), + (OBSERVABLE_DEPTH_LABEL, 'observable_depth', 'observable_depth_error'), + (PRIOR_OBSERVABLE_DEPTH_LABEL, 'prior_observable_depth', 'prior_observable_depth_error'), + (OBSERVABLE_DEPTH_DELTA_LABEL, 'observable_depth_prior_delta', 'observable_depth_prior_delta_error'), + ): + text = _format_depth(depth_summary.get(value_key), depth_summary.get(error_key)) + if text is not None: + entries[label] = text + return entries + + +def _depth_result_entry(value, error): + try: + value = float(value) + except (TypeError, ValueError): + return None + if not math.isfinite(value): + return None + try: + error = float(error) + except (TypeError, ValueError): + error = math.nan + if math.isfinite(error) and error >= 0: + value_text, uncertainty_text = format_value_and_uncertainty(value, error) + else: + value_text, uncertainty_text = str(round_to_2(value)), None + entry = {'value': value_text, 'units': "percent"} + if uncertainty_text is not None: + entry['uncertainty'] = uncertainty_text + return entry + + +def _result_entry(value, error, units=None): + value_text, uncertainty_text = format_value_and_uncertainty(value, error) + entry = {'value': value_text, 'uncertainty': uncertainty_text} + if units: + entry['units'] = units + return entry class OutputFiles: @@ -64,7 +165,12 @@ def __init__(self, fit, p_dict, i_dict, planetdir): self.dir = Path(planetdir) def final_lightcurve(self, phase): - params_file = self.dir / f"FinalLightCurve_{self.plname}_TESS.csv" + params_file = self.dir / safe_output_filename( + "FinalLightCurve", + self.plname, + "TESS", + extension="csv", + ) with params_file.open('w') as f: f.write(f"# FINAL TIMESERIES OF {self.p_dict['pl_name']}\n") @@ -76,26 +182,52 @@ def final_lightcurve(self, phase): f.write(f"{bjd}, {phase}, {flux}, {fluxerr}, {model}, {am}\n") def final_planetary_params(self, phot_opt, comp_star=None, comp_coords=None, min_aper=None, min_annul=None): - params_file = self.dir / f"FinalParams_{self.plname}_TESS.json" + params_file = self.dir / safe_output_filename( + "FinalParams", + self.plname, + "TESS", + extension="json", + ) params_num = { - "Mid-Transit Time (Tmid)": f"{round_to_2(self.fit.parameters['tmid'], self.fit.errors['tmid'])} +/- " - f"{round_to_2(self.fit.errors['tmid'])} BJD_TDB", - "Ratio of Planet to Stellar Radius (Rp/Rs)": f"{round_to_2(self.fit.parameters['rprs'], self.fit.errors['rprs'])} +/- " - f"{round_to_2(self.fit.errors['rprs'])}", - "Transit depth (Rp/Rs)^2": f"{round_to_2(100. * (self.fit.parameters['rprs'] ** 2.))} +/- " - f"{round_to_2(100. * 2. * self.fit.parameters['rprs'] * self.fit.errors['rprs'])} [%]", - "Semi Major Axis/Star Radius (a/Rs)": f"{round_to_2(self.fit.parameters['ars'], self.fit.errors['ars'])} +/- " - f"{round_to_2(self.fit.errors['ars'])} ", - "Airmass coefficient 1 (a1)": f"{round_to_2(self.fit.parameters['a1'], self.fit.errors['a1'])} +/- " - f"{round_to_2(self.fit.errors['a1'])}", - "Airmass coefficient 2 (a2)": f"{round_to_2(self.fit.parameters['a2'], self.fit.errors['a2'])} +/- " - f"{round_to_2(self.fit.errors['a2'])}", + "Mid-Transit Time (Tmid)": ( + f"{format_value_with_uncertainty(self.fit.parameters['tmid'], self.fit.errors['tmid'])} " + "BJD_TDB" + ), + "Ratio of Planet to Stellar Radius (Rp/Rs)": format_value_with_uncertainty( + self.fit.parameters['rprs'], self.fit.errors['rprs'] + ), + "Semi Major Axis/Star Radius (a/Rs)": ( + f"{format_value_with_uncertainty(self.fit.parameters['ars'], self.fit.errors['ars'])} " + ), + "Airmass coefficient 1 (a1)": format_value_with_uncertainty( + self.fit.parameters['a1'], self.fit.errors['a1'] + ), + "Airmass coefficient 2 (a2)": format_value_with_uncertainty( + self.fit.parameters['a2'], self.fit.errors['a2'] + ), "Scatter in the residuals of the lightcurve fit is": f"{round_to_2(100. * std(self.fit.residuals / median(self.fit.data)))} %", } + depth_params = _depth_final_params(self.fit, self.p_dict) + params_num = { + "Mid-Transit Time (Tmid)": params_num["Mid-Transit Time (Tmid)"], + "Ratio of Planet to Stellar Radius (Rp/Rs)": params_num["Ratio of Planet to Stellar Radius (Rp/Rs)"], + **depth_params, + "Semi Major Axis/Star Radius (a/Rs)": params_num["Semi Major Axis/Star Radius (a/Rs)"], + "Airmass coefficient 1 (a1)": params_num["Airmass coefficient 1 (a1)"], + "Airmass coefficient 2 (a2)": params_num["Airmass coefficient 2 (a2)"], + "Scatter in the residuals of the lightcurve fit is": params_num["Scatter in the residuals of the lightcurve fit is"], + } if phot_opt: - phot_ext = {"Best Comparison Star": f"#{comp_star} - {comp_coords}" if min_aper >= 0 else str(comp_star)} + transit_fit_comp_text = ( + f"#{comp_star} - {comp_coords}" + if comp_star is not None and min_aper >= 0 + else str(comp_star) + ) + phot_ext = { + "Transit Fit Comparison Star": transit_fit_comp_text + } if min_aper == 0: phot_ext["Optimal Method"] = "PSF photometry" else: @@ -103,8 +235,9 @@ def final_planetary_params(self, phot_opt, comp_star=None, comp_coords=None, min phot_ext["Optimal Annulus"] = f"{min_annul}" params_num.update(phot_ext) - params_num["Transit Duration (day)"] = (f"{round_to_2(mean(self.durs), std(self.durs))} +/- " - f"{round_to_2(std(self.durs))}") + params_num["Transit Duration (day)"] = format_value_with_uncertainty( + mean(self.durs), std(self.durs) + ) final_params = {'FINAL PLANETARY PARAMETERS': params_num} with params_file.open('w') as f: @@ -113,13 +246,25 @@ def final_planetary_params(self, phot_opt, comp_star=None, comp_coords=None, min def aavso(self, airmasses, ld0, ld1, ld2, ld3, tmidstr): priors_dict, filter_dict, results_dict = aavso_dicts(self.p_dict, self.fit, self.i_dict, self.durs, ld0, ld1, ld2, ld3) + gaia_dist = "" if self.p_dict.get('dist') is None else str(self.p_dict.get('dist')) + gaia_pmra = "" if self.p_dict.get('pm_ra') is None else str(self.p_dict.get('pm_ra')) + gaia_pmdec = "" if self.p_dict.get('pm_dec') is None else str(self.p_dict.get('pm_dec')) + gaia_dist_header = f"#GAIADIST={gaia_dist}\n" if gaia_dist else "" + gaia_pmra_header = f"#GAIAPMRA={gaia_pmra}\n" if gaia_pmra else "" + gaia_pmdec_header = f"#GAIAPMDEC={gaia_pmdec}\n" if gaia_pmdec else "" # compute 32 character hash of the results_dict hash_object = hashlib.sha256(dumps(results_dict).encode()) hash_id = hash_object.hexdigest()[:32] #params_file = self.dir / f"TESS_{hash_id}_{self.plname}_{tmidstr}_AAVSO.txt" - params_file = self.dir / f"{tmidstr}_{hash_id}_{self.plname}_AAVSO.txt" + params_file = aavso_output_directory(self.dir) / safe_output_filename( + tmidstr, + hash_id, + self.plname, + "AAVSO", + extension="txt", + ) # 2459642_61_5164d266e1755aead98dbec0f26e7b7c_gj436b_AAVSO with params_file.open('w') as f: @@ -131,26 +276,29 @@ def aavso(self, airmasses, ld0, ld1, ld2, ld3, tmidstr): "#DATE_TYPE=BJD_TDB\n" # fixed f"#OBSTYPE=CCD\n" f"#STAR_NAME={self.p_dict['hostname']}\n" # code yields - f"#EXOPLANET_NAME={self.p_dict['pl_name']}\n" # code yields + f"#EXOPLANET_NAME={format_aavso_exoplanet_name(self.p_dict['pl_name'])}\n" # code yields f"#BINNING=1x1\n" # uhhh i just put One. f"#EXPOSURE_TIME={self.i_dict.get('exposure', -1)}\n" # UI + f"{gaia_dist_header}" + f"{gaia_pmra_header}" + f"{gaia_pmdec_header}" f"#COMP_STAR-XC=null\n" f"#NOTES=TESS Data\n" "#DETREND_PARAMETERS=AIRMASS, AIRMASS CORRECTION FUNCTION\n" # fixed "#MEASUREMENT_TYPE=Rnflux\n" # fixed f"#FILTER=I\n" f"#FILTER-XC={dumps(filter_dict)}\n" - f"#PRIORS=Period={round_to_2(self.p_dict['pl_orbper'], self.p_dict['pl_orbpererr1'])} +/- {round_to_2(self.p_dict['pl_orbpererr1'])}" - f",a/R*={round_to_2(self.p_dict['pl_ratdor'], self.p_dict['pl_ratdorerr1'])} +/- {round_to_2(self.p_dict['pl_ratdorerr1'])}" - f",inc={round_to_2(self.p_dict['pl_orbincl'], self.p_dict['pl_orbinclerr1'])} +/- {round_to_2(self.p_dict['pl_orbinclerr1'])}" + f"#PRIORS=Period={format_value_with_uncertainty(self.p_dict['pl_orbper'], self.p_dict['pl_orbpererr1'])}" + f",a/R*={format_value_with_uncertainty(self.p_dict['pl_ratdor'], self.p_dict['pl_ratdorerr1'])}" + f",inc={format_value_with_uncertainty(self.p_dict['pl_orbincl'], self.p_dict['pl_orbinclerr1'])}" f",ecc={round_to_2(self.p_dict['pl_orbeccen'])}" f",u0={round_to_2(ld0)}" f",u1={round_to_2(ld1)}" f",u2={round_to_2(ld2)}" f",u3={round_to_2(ld3)}\n" f"#PRIORS-XC={dumps(priors_dict)}\n" # code yields - f"#RESULTS=Tc={round_to_2(self.fit.parameters['tmid'], self.fit.errors['tmid'])} +/- {round_to_2(self.fit.errors['tmid'])}" - f",Rp/R*={round_to_2(self.fit.parameters['rprs'], self.fit.errors['rprs'])} +/- {round_to_2(self.fit.errors['rprs'])}" + f"#RESULTS=Tc={format_value_with_uncertainty(self.fit.parameters['tmid'], self.fit.errors['tmid'])}" + f",Rp/R*={format_value_with_uncertainty(self.fit.parameters['rprs'], self.fit.errors['rprs'])}" f",Am1=0" f",Am2=0\n" f"#RESULTS-XC={dumps(results_dict)}\n") # code yields @@ -176,8 +324,20 @@ def aavso(self, airmasses, ld0, ld1, ld2, ld3, tmidstr): def aavso_csv(self, airmasses, ld0, ld1, ld2, ld3,tmidstr): priors_dict, filter_dict, results_dict = aavso_dicts(self.p_dict, self.fit, self.i_dict, self.durs, ld0, ld1, ld2, ld3) + gaia_dist = "" if self.p_dict.get('dist') is None else str(self.p_dict.get('dist')) + gaia_pmra = "" if self.p_dict.get('pm_ra') is None else str(self.p_dict.get('pm_ra')) + gaia_pmdec = "" if self.p_dict.get('pm_dec') is None else str(self.p_dict.get('pm_dec')) + gaia_dist_header = f"#GAIADIST={gaia_dist}\n" if gaia_dist else "" + gaia_pmra_header = f"#GAIAPMRA={gaia_pmra}\n" if gaia_pmra else "" + gaia_pmdec_header = f"#GAIAPMDEC={gaia_pmdec}\n" if gaia_pmdec else "" - params_file = self.dir / f"TESS_{tmidstr}_{self.p_dict['pl_name']}_lightcurve.csv" + params_file = aavso_output_directory(self.dir) / safe_output_filename( + "TESS", + tmidstr, + self.p_dict['pl_name'], + "lightcurve", + extension="csv", + ) with params_file.open('w') as f: f.write("#TYPE=EXOPLANET\n" # fixed @@ -188,26 +348,29 @@ def aavso_csv(self, airmasses, ld0, ld1, ld2, ld3,tmidstr): "#DATE_TYPE=BJD_TDB\n" # fixed f"#OBSTYPE=CCD\n" f"#STAR_NAME={self.p_dict['hostname']}\n" # code yields - f"#EXOPLANET_NAME={self.p_dict['pl_name']}\n" # code yields + f"#EXOPLANET_NAME={format_aavso_exoplanet_name(self.p_dict['pl_name'])}\n" # code yields f"#BINNING=1x1\n" # uhhh i just put One. f"#EXPOSURE_TIME={self.i_dict.get('exposure', -1)}\n" # UI + f"{gaia_dist_header}" + f"{gaia_pmra_header}" + f"{gaia_pmdec_header}" f"#COMP_STAR-XC=null\n" f"#NOTES=TESS Data\n" "#DETREND_PARAMETERS=AIRMASS, AIRMASS CORRECTION FUNCTION\n" # fixed "#MEASUREMENT_TYPE=Rnflux\n" # fixed f"#FILTER=I\n" f"#FILTER-XC={dumps(filter_dict)}\n" - f"#PRIORS=Period={round_to_2(self.p_dict['pl_orbper'], self.p_dict['pl_orbpererr1'])} +/- {round_to_2(self.p_dict['pl_orbpererr1'])}" - f",a/R*={round_to_2(self.p_dict['pl_ratdor'], self.p_dict['pl_ratdorerr1'])} +/- {round_to_2(self.p_dict['pl_ratdorerr1'])}" - f",inc={round_to_2(self.p_dict['pl_orbincl'], self.p_dict['pl_orbinclerr1'])} +/- {round_to_2(self.p_dict['pl_orbinclerr1'])}" + f"#PRIORS=Period={format_value_with_uncertainty(self.p_dict['pl_orbper'], self.p_dict['pl_orbpererr1'])}" + f",a/R*={format_value_with_uncertainty(self.p_dict['pl_ratdor'], self.p_dict['pl_ratdorerr1'])}" + f",inc={format_value_with_uncertainty(self.p_dict['pl_orbincl'], self.p_dict['pl_orbinclerr1'])}" f",ecc={round_to_2(self.p_dict['pl_orbeccen'])}" f",u0={round_to_2(ld0)}" f",u1={round_to_2(ld1)}" f",u2={round_to_2(ld2)}" f",u3={round_to_2(ld3)}\n" f"#PRIORS-XC={dumps(priors_dict)}\n" # code yields - f"#RESULTS=Tc={round_to_2(self.fit.parameters['tmid'], self.fit.errors['tmid'])} +/- {round_to_2(self.fit.errors['tmid'])}" - f",Rp/R*={round_to_2(self.fit.parameters['rprs'], self.fit.errors['rprs'])} +/- {round_to_2(self.fit.errors['rprs'])}" + f"#RESULTS=Tc={format_value_with_uncertainty(self.fit.parameters['tmid'], self.fit.errors['tmid'])}" + f",Rp/R*={format_value_with_uncertainty(self.fit.parameters['rprs'], self.fit.errors['rprs'])}" f",Am1=0" f",Am2=0\n" f"#RESULTS-XC={dumps(results_dict)}\n") # code yields @@ -232,20 +395,13 @@ def aavso_csv(self, airmasses, ld0, ld1, ld2, ld3,tmidstr): def aavso_dicts(planet_dict, fit, i_dict, durs, ld0, ld1, ld2, ld3): priors = { - 'Period': { - 'value': str(round_to_2(planet_dict['pl_orbper'], planet_dict['pl_orbpererr1'])), - 'uncertainty': str(round_to_2(planet_dict['pl_orbpererr1'])) if planet_dict['pl_orbpererr1'] else None, - 'units': "days" - }, - 'a/R*': { - 'value': str(round_to_2(planet_dict['pl_ratdor'], planet_dict['pl_ratdorerr1'])), - 'uncertainty': str(round_to_2(planet_dict['pl_ratdorerr1'])) if planet_dict['pl_ratdorerr1'] else planet_dict['pl_ratdorerr1'], - }, - 'inc': { - 'value': str(round_to_2(planet_dict['pl_orbincl'], planet_dict['pl_orbinclerr1'])), - 'uncertainty': str(round_to_2(planet_dict['pl_orbinclerr1'])) if planet_dict['pl_orbinclerr1'] else planet_dict['pl_orbinclerr1'], - 'units': "degrees" - }, + 'Period': _result_entry( + planet_dict['pl_orbper'], planet_dict['pl_orbpererr1'], units="days" + ), + 'a/R*': _result_entry(planet_dict['pl_ratdor'], planet_dict['pl_ratdorerr1']), + 'inc': _result_entry( + planet_dict['pl_orbincl'], planet_dict['pl_orbinclerr1'], units="degrees" + ), 'ecc': { 'value': str(round_to_2(planet_dict['pl_orbeccen'])), 'uncertainty': None, @@ -270,20 +426,21 @@ def aavso_dicts(planet_dict, fit, i_dict, durs, ld0, ld1, ld2, ld3): filter_type = { 'name': "I", - 'fwhm': [{'value': 600, 'units': "nm"}, - {'value': 1000, 'units': "nm"}], + 'filter_width': { + 'left_side_wavelength': { + 'value': 600, + 'units': "nm" + }, + 'right_side_wavelength': { + 'value': 1000, + 'units': "nm" + } + }, } results = { - 'Tc': { - 'value': str(round_to_2(fit.parameters['tmid'], fit.errors['tmid'])), - 'uncertainty': str(round_to_2(fit.errors['tmid'])), - 'units': "BJD_TDB" - }, - 'Rp/R*': { - 'value': str(round_to_2(fit.parameters['rprs'], fit.errors['rprs'])), - 'uncertainty': str(round_to_2(fit.errors['rprs'])) - }, + 'Tc': _result_entry(fit.parameters['tmid'], fit.errors['tmid'], units="BJD_TDB"), + 'Rp/R*': _result_entry(fit.parameters['rprs'], fit.errors['rprs']), 'Am1': { 'value': 0, 'uncertainty': None @@ -292,26 +449,37 @@ def aavso_dicts(planet_dict, fit, i_dict, durs, ld0, ld1, ld2, ld3): 'value': 0, 'uncertainty': None }, - 'Duration': { - 'value': str(round_to_2(mean(durs))), - 'uncertainty': str(round_to_2(std(durs))), - 'units': "days" - } + 'Duration': _result_entry(mean(durs), std(durs), units="days"), } # try to add a/Rs if it exists if 'ars' in fit.errors: - results['a/R*'] = { - 'value': str(round_to_2(fit.parameters['ars'], fit.errors['ars'])), - 'uncertainty': str(round_to_2(fit.errors['ars'])), - } + results['a/R*'] = _result_entry(fit.parameters['ars'], fit.errors['ars']) # check for inclination if 'inc' in fit.errors: - results['inc'] = { - 'value': str(round_to_2(fit.parameters['inc'], fit.errors['inc'])), - 'uncertainty': str(round_to_2(fit.errors['inc'])), - 'units': "degrees" - } + results['inc'] = _result_entry( + fit.parameters['inc'], fit.errors['inc'], units="degrees" + ) + + limb_darkening = (ld0, ld1, ld2, ld3) + depth_summary = fit_transit_depth_summary( + fit, + prior_parameters=planet_dict_transit_parameters( + planet_dict, + limb_darkening=limb_darkening, + fallback=getattr(fit, 'prior', None), + ), + prior_errors=planet_dict_transit_errors(planet_dict, limb_darkening=limb_darkening), + ) + for label, value_key, error_key in ( + (AREA_DEPTH_LABEL, 'area_depth', 'area_depth_error'), + (OBSERVABLE_DEPTH_LABEL, 'observable_depth', 'observable_depth_error'), + (PRIOR_OBSERVABLE_DEPTH_LABEL, 'prior_observable_depth', 'prior_observable_depth_error'), + (OBSERVABLE_DEPTH_DELTA_LABEL, 'observable_depth_prior_delta', 'observable_depth_prior_delta_error'), + ): + entry = _depth_result_entry(depth_summary.get(value_key), depth_summary.get(error_key)) + if entry is not None: + results[label] = entry return priors, filter_type, results diff --git a/exotic/api/plate_solution.py b/exotic/api/plate_solution.py index 68f6be5b..c1d1294d 100644 --- a/exotic/api/plate_solution.py +++ b/exotic/api/plate_solution.py @@ -35,17 +35,39 @@ # EXOplanet Transit Interpretation Code (EXOTIC) # # NOTE: See companion file version.py for version info. # ########################################################################### # -from astropy.io.fits import PrimaryHDU, getdata, getheader +from astropy.io.fits import Header, PrimaryHDU, getdata, getheader +from astropy.stats import sigma_clipped_stats from json import dumps from pathlib import Path +import numpy as np +from photutils.detection import DAOStarFinder import requests +import time from tenacity import retry, retry_if_exception_type, retry_if_result, \ stop_after_attempt, wait_exponential +try: + from .http_compression import build_compressed_json_request +except ImportError: + from http_compression import build_compressed_json_request + +try: + from ..version import __version__ +except ImportError: + try: + from version import __version__ + except ImportError: + __version__ = "unknown" + _R_MAX_STOPS_LOW = 7 _R_MAX_STOPS = 10 _R_MAX_SECS = 37 _RQ_TIMEOUT = 16.0 +_NEXTASTRO_MAX_SOURCES = 200 +_NEXTASTRO_STATUS_MAX_POLLS = 60 +_NEXTASTRO_STATUS_POLL_SEC = 2 +_NEXTASTRO_IN_PROGRESS_STATUSES = {'queued', 'running'} +_NEXTASTRO_SOFTWARE_NAME = f"EXOTIC/{__version__}" def is_false(value): @@ -59,34 +81,43 @@ def result_if_max_retry_count(retry_state): class PlateSolution: def __init__(self, file=None, directory=None, api_key=None, - api_url='http://nova.astrometry.net/api/'): + api_url='http://nova.astrometry.net/api/', ra=None, dec=None, + pixel_scale=None, radius=2.0, scale_err=25.0, suppress_fail_warning=False): if api_key is None: api_key = {'apikey': 'vfsyxlmdxfryhprq'} self.api_url = api_url self.api_key = api_key self.file = file self.directory = directory + self.ra = ra + self.dec = dec + self.pixel_scale = pixel_scale + self.radius = radius + self.scale_err = scale_err + self.suppress_fail_warning = suppress_fail_warning + self.last_error_type = None def plate_solution(self): + self.last_error_type = None session = self._login() if not session: - return PlateSolution.fail('Login') + return self._fail('Login') sub_id = self._upload(session) if not sub_id: - return PlateSolution.fail('Upload') + return self._fail('Upload') sub_url = self._get_url(f"submissions/{sub_id}") job_id = self._sub_status(sub_url) if not job_id: - return PlateSolution.fail('Submission ID') + return self._fail('Submission ID') job_url = self._get_url(f"jobs/{job_id}") download_url = self.api_url.replace("/api/", f"/wcs_file/{job_id}/") - wcs_file = Path(self.directory) / "temp" / "wcs.fits" + wcs_file = Path(self.directory) / "working_artifacts" / "wcs.fits" wcs_file = self._job_status(job_url, wcs_file, download_url) if not wcs_file: - return PlateSolution.fail('Job Status') + return self._fail('Job Status') else: print("WCS file creation successful.") return wcs_file @@ -94,6 +125,12 @@ def plate_solution(self): def _get_url(self, service): return self.api_url + service + def _fail(self, error_type, service_name='nova.astrometry.net'): + self.last_error_type = error_type + if self.suppress_fail_warning: + return False + return PlateSolution.fail(error_type, service_name=service_name) + @retry(stop=stop_after_attempt(_R_MAX_STOPS_LOW), wait=wait_exponential(multiplier=1, min=4, max=_R_MAX_SECS), retry=(retry_if_result(is_false) | retry_if_exception_type(requests.exceptions.RequestException)), retry_error_callback=result_if_max_retry_count) @@ -109,11 +146,29 @@ def _login(self): retry=(retry_if_result(is_false) | retry_if_exception_type(requests.exceptions.RequestException)), retry_error_callback=result_if_max_retry_count) def _upload(self, session): - files = {'file': open(self.file, 'rb')} - headers = {'request-json': dumps({"session": session}), 'allow_commercial_use': 'n', + request_payload = {"session": session} + + if self.ra is not None and self.dec is not None: + request_payload.update({ + "center_ra": float(self.ra), + "center_dec": float(self.dec), + "radius": float(self.radius) + }) + + if self.pixel_scale not in (None, ""): + request_payload.update({ + "scale_units": "arcsecperpix", + "scale_type": "ev", + "scale_est": float(self.pixel_scale), + "scale_err": float(self.scale_err) + }) + + headers = {'request-json': dumps(request_payload), 'allow_commercial_use': 'n', 'allow_modifications': 'n', 'publicly_visible': 'n'} - r = requests.post(self.api_url + 'upload', files=files, data=headers, timeout=_RQ_TIMEOUT) + with open(self.file, 'rb') as image_file: + files = {'file': image_file} + r = requests.post(self.api_url + 'upload', files=files, data=headers, timeout=_RQ_TIMEOUT) if r.json()['status'] == 'success': return r.json()['subid'] @@ -143,7 +198,258 @@ def _job_status(self, job_url, wcs_file, download_url): return False @staticmethod - def fail(error_type): - print("WARNING: After multiple attempts, EXOTIC could not retrieve a plate solution from nova.astrometry.net" - f" due to {error_type}. EXOTIC will continue reducing data without a plate solution.") + def fail(error_type, service_name='nova.astrometry.net'): + print("WARNING: After multiple attempts, EXOTIC could not retrieve a plate solution from " + f"{service_name} due to {error_type}. EXOTIC will continue reducing data without a plate solution.") + return False + + +class NextAstroPlateSolution: + + def __init__(self, file=None, directory=None, api_url='https://astrometry.nextastro.org/', ra=None, dec=None, + pixel_scale=None, suppress_fail_warning=False, message_logger=None): + self.api_url = api_url.rstrip('/') + self.file = file + self.directory = directory + self.ra = ra + self.dec = dec + self.pixel_scale = pixel_scale + self.suppress_fail_warning = suppress_fail_warning + self.message_logger = message_logger + self.last_error_type = None + self.last_http_status = None + + def plate_solution(self): + self.last_error_type = None + self.last_http_status = None + self._emit_debug(f"Using NextAstro astrometry server at {self.api_url} for plate solving.") + source_list = self._generate_source_list() + if not source_list: + return self._fail('Source extraction for NextAstro astrometry server') + + request_id = self._submit_solve_request(source_list) + if not request_id: + return self._fail('NextAstro solve submission') + + wcs_header = self._poll_for_solution(request_id) + if not wcs_header: + return self._fail('NextAstro solve status') + + wcs_file = Path(self.directory) / "working_artifacts" / "wcs.fits" + hdu = PrimaryHDU(data=getdata(filename=self.file), header=wcs_header) + hdu.writeto(wcs_file, overwrite=True) + self._emit_debug("WCS file creation successful.") + return wcs_file + + def _emit_debug(self, message): + if self.message_logger is not None: + self.message_logger(message) + elif not self.suppress_fail_warning: + print(message) + + @staticmethod + def _json_message(payload): + try: + return dumps(payload) + except (TypeError, ValueError): + return str(payload) + + def _fail(self, error_type): + self.last_error_type = error_type + if self.suppress_fail_warning: + return False + return PlateSolution.fail(error_type, service_name=f'NextAstro ({self.api_url})') + + def _generate_source_list(self): + image_data = np.asarray(getdata(filename=self.file), dtype=float) + if image_data.ndim > 2: + image_data = image_data.squeeze() + + median, _, std = sigma_clipped_stats(image_data, sigma=3.0) + if std <= 0: + std = float(np.nanstd(image_data)) + if std <= 0: + return None + + finder = DAOStarFinder(fwhm=3.0, threshold=3.5 * std) + sources = finder(image_data - median) + + if sources is not None and len(sources) > 0: + bright_sources = self._limit_to_brightest_sources( + x_coords=sources['xcentroid'], + y_coords=sources['ycentroid'], + fluxes=sources['flux'] + ) + if bright_sources is None: + return None + return { + "x": bright_sources["x"], + "y": bright_sources["y"], + "flux": bright_sources["flux"], + "origin": "exotic", + "pixel_indexing": "0-based" + } + + return self._fallback_source_list(image_data, median, std) + + + def _fallback_source_list(self, image_data, median, std): + threshold = median + 3.5 * std + candidate_indices = np.argwhere(image_data > threshold) + if candidate_indices.size == 0: + return None + + candidate_fluxes = image_data[candidate_indices[:, 0], candidate_indices[:, 1]] + bright_sources = self._limit_to_brightest_sources( + x_coords=candidate_indices[:, 1], + y_coords=candidate_indices[:, 0], + fluxes=candidate_fluxes + ) + if bright_sources is None: + return None + + return { + "x": bright_sources["x"], + "y": bright_sources["y"], + "flux": bright_sources["flux"], + "origin": "exotic", + "pixel_indexing": "0-based" + } + + @staticmethod + def _limit_to_brightest_sources(x_coords, y_coords, fluxes): + fluxes = np.asarray(fluxes, dtype=float) + x_coords = np.asarray(x_coords, dtype=float) + y_coords = np.asarray(y_coords, dtype=float) + + finite_flux_mask = np.isfinite(fluxes) + if not np.any(finite_flux_mask): + return None + + fluxes = fluxes[finite_flux_mask] + x_coords = x_coords[finite_flux_mask] + y_coords = y_coords[finite_flux_mask] + + sorted_indices = np.argsort(fluxes)[::-1][:_NEXTASTRO_MAX_SOURCES] + + return { + "x": x_coords[sorted_indices].tolist(), + "y": y_coords[sorted_indices].tolist(), + "flux": fluxes[sorted_indices].tolist() + } + + @staticmethod + def _response_body_preview(response, max_chars=240): + body = getattr(response, 'text', None) + if body is None: + content = getattr(response, 'content', b'') + body = content.decode(errors='replace') if isinstance(content, bytes) else str(content) + + body = " ".join(str(body).split()) + if not body: + return "" + if len(body) > max_chars: + return body[:max_chars - 3] + "..." + return body + + def _decode_response_json(self, response, context): + self.last_http_status = getattr(response, 'status_code', None) + try: + return response.json() + except ValueError: + if response.status_code != 502: + self._emit_debug(f"[NextAstro] {context} returned non-JSON response " + f"(HTTP {response.status_code}): {self._response_body_preview(response)}") + return None + + def _submit_solve_request(self, source_list): + image_data = getdata(filename=self.file) + payload = { + "sources": source_list, + "image": { + "width": int(image_data.shape[-1]), + "height": int(image_data.shape[-2]), + }, + "options": { + "timeout_sec": 120, + "max_sources": _NEXTASTRO_MAX_SOURCES + } + } + + hints = self._extract_astrometry_hints() + if hints is not None: + payload["hints"] = hints + + request_body, headers, content_encoding, raw_size, compressed_size = build_compressed_json_request(payload) + headers["X-NextAstro-Software"] = _NEXTASTRO_SOFTWARE_NAME + + self._emit_debug(f"NextAstro astrometry request JSON: {self._json_message(payload)}") + self._emit_debug( + "NextAstro astrometry request compression: " + f"{content_encoding} ({compressed_size} bytes sent; {raw_size} bytes raw)" + ) + response = requests.post(f"{self.api_url}/solve", data=request_body, headers=headers, timeout=_RQ_TIMEOUT) + response_json = self._decode_response_json(response, 'Solve response') + if response_json is not None and response.status_code != 502: + self._emit_debug(f"NextAstro astrometry submission response JSON: {self._json_message(response_json)}") + if response.status_code >= 400 or response_json is None: + return False + if response_json.get('status') in {'queued', 'running'}: + return response_json.get('request_id') + return False + + def _extract_astrometry_hints(self): + hints = {} + + if self.ra is not None and self.dec is not None: + hints.update({"ra_deg": self.ra, "dec_deg": self.dec}) + + if self.pixel_scale not in (None, ""): + hints.update({"scale_arcsec_per_pix": float(self.pixel_scale), "scale_tolerance_frac": 0.25}) + + if not hints: + return None + + return hints + + def _poll_for_solution(self, request_id): + latest_status = None + for _ in range(_NEXTASTRO_STATUS_MAX_POLLS): + response = requests.get(f"{self.api_url}/status/{request_id}", timeout=_RQ_TIMEOUT) + response_json = self._decode_response_json(response, 'Status response') + if response_json is None: + return False + if response.status_code >= 400: + if response.status_code != 502: + self._emit_debug( + f"NextAstro astrometry status response JSON (HTTP {response.status_code}): " + f"{self._json_message(response_json)}" + ) + return False + + status = str(response_json.get('status', '')).lower() + latest_status = response_json.get('status') + if status == 'solved': + self._emit_debug( + f"NextAstro astrometry status response JSON (solved): {self._json_message(response_json)}" + ) + header_dict = response_json.get('solution', {}).get('wcs_header') + if isinstance(header_dict, dict): + return Header(header_dict) + return False + if status == 'failed': + self._emit_debug( + f"NextAstro astrometry status response JSON (failed): {self._json_message(response_json)}" + ) + return False + + if status not in _NEXTASTRO_IN_PROGRESS_STATUSES: + self._emit_debug( + f"NextAstro astrometry status response JSON (unexpected): {self._json_message(response_json)}" + ) + return False + + time.sleep(_NEXTASTRO_STATUS_POLL_SEC) + + self._emit_debug(f"[NextAstro] Polling timed out waiting for terminal status; latest status={latest_status!r}") return False diff --git a/exotic/api/plotting.py b/exotic/api/plotting.py index 35f324ab..9bce6220 100644 --- a/exotic/api/plotting.py +++ b/exotic/api/plotting.py @@ -35,16 +35,13 @@ # EXOplanet Transit Interpretation Code (EXOTIC) # # NOTE: See companion file version.py for version info. # ########################################################################### # -from astropy.io import fits -# from astroscrappy import detect_cosmics -from bokeh.io import output_notebook -from bokeh.models import BoxZoomTool, ColorBar, FreehandDrawTool, HoverTool, LinearColorMapper, LogColorMapper, \ - LogTicker, PanTool, ResetTool, WheelZoomTool -from bokeh.palettes import Viridis256 -from bokeh.plotting import figure, output_file, show -from io import BytesIO -import json -import logging +from astropy.io import fits +# from astroscrappy import detect_cosmics +from bokeh.models import BoxZoomTool, ColorBar, FreehandDrawTool, HoverTool, LogColorMapper, \ + LogTicker, PanTool, ResetTool, WheelZoomTool +from bokeh.plotting import figure, output_file, show +import json +import logging import matplotlib.pyplot as plt from matplotlib.ticker import MaxNLocator, NullLocator, ScalarFormatter import numpy as np @@ -431,7 +428,7 @@ def corner(xs, bins=20, range=None, weights=None, color="k", hist_bin_factor=1, return fig -def quantile(x, q, weights=None): +def quantile(x, q, weights=None): """ Compute sample quantiles with support for weighted samples. @@ -478,15 +475,44 @@ def quantile(x, q, weights=None): raise ValueError("Dimension mismatch: len(weights) != len(x)") idx = np.argsort(x) sw = weights[idx] - cdf = np.cumsum(sw)[:-1] - cdf /= cdf[-1] - cdf = np.append(0, cdf) - return np.interp(q, cdf, x[idx]).tolist() - -def hist2d(x, y, bins=20, range=None, levels=[2], - ax=None, plot_datapoints=True, plot_contours=True, - contour_kwargs=None, contourf_kwargs=None, data_kwargs=None, - **kwargs): + cdf = np.cumsum(sw)[:-1] + cdf /= cdf[-1] + cdf = np.append(0, cdf) + return np.interp(q, cdf, x[idx]).tolist() + +def _contour_levels_within_surface(levels, surface_min, surface_max): + levels = np.asarray(levels, dtype=float) + levels = np.unique(levels[np.isfinite(levels)]) + if levels.size == 0: + return levels + + if ( + not np.isfinite(surface_min) + or not np.isfinite(surface_max) + or surface_min >= surface_max + ): + return np.array([], dtype=float) + + valid_levels = levels[(levels > surface_min) & (levels < surface_max)] + if valid_levels.size > 0: + return valid_levels + + surface_span = float(surface_max - surface_min) + epsilon = max( + surface_span * 1e-9, + np.finfo(float).eps * max(1.0, abs(float(surface_min)), abs(float(surface_max))), + ) + lower = float(surface_min) + epsilon + upper = float(surface_max) - epsilon + if not np.isfinite(lower) or not np.isfinite(upper) or lower >= upper: + return np.array([0.5 * (float(surface_min) + float(surface_max))], dtype=float) + + return np.unique(np.clip(levels, lower, upper)) + +def hist2d(x, y, bins=20, range=None, levels=[2], + ax=None, plot_datapoints=True, plot_contours=True, + contour_kwargs=None, contourf_kwargs=None, data_kwargs=None, + **kwargs): if ax is None: ax = plt.gca() @@ -497,26 +523,48 @@ def hist2d(x, y, bins=20, range=None, levels=[2], data_kwargs["alpha"] = data_kwargs.get("alpha", 0.2) ax.scatter(x, y, marker="o", zorder=-1, rasterized=True, **data_kwargs) - # Plot the contour edge colors. - if plot_contours: - if contour_kwargs is None: - contour_kwargs = dict() - - # mask data in range + chi2 - maskx = (x > range[0][0]) & (x < range[0][1]) - masky = (y > range[1][0]) & (y < range[1][1]) - mask = maskx & masky & (data_kwargs['c'] < data_kwargs['vmax']*1.2) - - try: # contour - # approx posterior + smooth - xg, yg = np.meshgrid( np.linspace(x[mask].min(),x[mask].max(),256), np.linspace(y[mask].min(),y[mask].max(),256) ) - cg = griddata(np.vstack([x[mask],y[mask]]).T, data_kwargs['c'][mask], (xg,yg), method='nearest', rescale=True) - scg = gaussian_filter(cg,sigma=15) - - ax.contour(xg, yg, scg*np.nanmin(cg)/np.nanmin(scg), np.sort(levels), **contour_kwargs, vmin=data_kwargs['vmin'], vmax=data_kwargs['vmax']) - except Exception as err: - print(err) - print("contour plotting failed") + # Plot the contour edge colors. + if plot_contours: + if contour_kwargs is None: + contour_kwargs = dict() + + contour_levels = np.asarray(levels, dtype=float) + contour_levels = np.unique(contour_levels[np.isfinite(contour_levels)]) + if contour_levels.size == 0: + ax.set_xlim(range[0]) + ax.set_ylim(range[1]) + return + + # mask data in range + chi2 + maskx = (x > range[0][0]) & (x < range[0][1]) + masky = (y > range[1][0]) & (y < range[1][1]) + mask = maskx & masky & (data_kwargs['c'] < data_kwargs['vmax']*1.2) + + try: # contour + if np.count_nonzero(mask) < 3: + raise ValueError("not enough in-range samples for contour plotting") + # approx posterior + smooth + xg, yg = np.meshgrid( np.linspace(x[mask].min(),x[mask].max(),256), np.linspace(y[mask].min(),y[mask].max(),256) ) + cg = griddata(np.vstack([x[mask],y[mask]]).T, data_kwargs['c'][mask], (xg,yg), method='nearest', rescale=True) + scg = gaussian_filter(cg,sigma=15) + cg_min = np.nanmin(cg) + scg_min = np.nanmin(scg) + if not np.isfinite(cg_min) or not np.isfinite(scg_min) or np.isclose(scg_min, 0): + raise ValueError("degenerate contour surface") + + contour_surface = scg * cg_min / scg_min + surface_min = np.nanmin(contour_surface) + surface_max = np.nanmax(contour_surface) + if not np.isfinite(surface_min) or not np.isfinite(surface_max) or np.isclose(surface_min, surface_max): + raise ValueError("degenerate contour surface") + contour_levels = _contour_levels_within_surface(contour_levels, surface_min, surface_max) + if contour_levels.size == 0: + raise ValueError("no contour levels fall within the plotted surface") + + ax.contour(xg, yg, contour_surface, contour_levels, **contour_kwargs, vmin=data_kwargs['vmin'], vmax=data_kwargs['vmax']) + except Exception as err: + print(err) + print("contour plotting failed") ax.set_xlim(range[0]) ax.set_ylim(range[1]) diff --git a/exotic/api/rv_fitter.py b/exotic/api/rv_fitter.py index af4fa742..92e8b058 100644 --- a/exotic/api/rv_fitter.py +++ b/exotic/api/rv_fitter.py @@ -43,13 +43,17 @@ import matplotlib.pyplot as plt import numpy as np from ultranest import ReactiveNestedSampler -from scipy.optimize import least_squares try: from elca import lc_fitter except ImportError: from .elca import lc_fitter +try: + from ultranest_utils import run_reactive_sampler +except ImportError: + from .ultranest_utils import run_reactive_sampler + Mjup = const.M_jup.to(u.kg).value Msun = const.M_sun.to(u.kg).value @@ -261,10 +265,12 @@ def loglike(pars): for k in lfreekeys[n]: freekeys.append(f"local_{n}_{k}") - if self.verbose: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=6e5) - else: - self.results = ReactiveNestedSampler(freekeys, loglike, prior_transform).run(max_ncalls=6e5, show_status=self.verbose, viz_callback=self.verbose) + sampler = ReactiveNestedSampler(freekeys, loglike, prior_transform) + self.results = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": int(6e5)}, + verbose=self.verbose, + ) self.parameters = {} self.quantiles = {} @@ -593,4 +599,4 @@ def plot_orbit(self): return fig - \ No newline at end of file + diff --git a/exotic/api/ultranest_utils.py b/exotic/api/ultranest_utils.py new file mode 100644 index 00000000..c9aa9006 --- /dev/null +++ b/exotic/api/ultranest_utils.py @@ -0,0 +1,729 @@ +import gc +import logging +import math +import multiprocessing +import os +import sys +import time +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager + +import numpy as np + + +_TRUTHY = {"1", "true", "yes", "on"} +_FALSEY = {"0", "false", "no", "off", "n"} +_AUTO_WORKER_VALUES = {"auto", "all", "available", "cpu", "cpus", "core", "cores"} +DEFAULT_PROGRESS_INTERVAL_SECONDS = 10.0 +DEFAULT_MIN_NUM_LIVE_POINTS = 200 +DEFAULT_RUN_KWARGS = { + "min_num_live_points": DEFAULT_MIN_NUM_LIVE_POINTS, + "min_ess": 200, + "dlogz": 1.0, + "dKL": 1.0, + "frac_remain": 0.05, + "max_num_improvement_loops": 1, +} +MIN_LIVE_POINTS_ENV_KEYS = ( + "EXOTIC_ULTRANEST_MIN_NUM_LIVE_POINTS", + "EXOTIC_ULTRANEST_MIN_LIVE_POINTS", +) +MPI_SIZE_ENV_KEYS = ( + "OMPI_COMM_WORLD_SIZE", + "PMI_SIZE", + "PMIX_SIZE", + "MV2_COMM_WORLD_SIZE", +) +MPI_RANK_ENV_KEYS = ( + "OMPI_COMM_WORLD_RANK", + "PMI_RANK", + "PMIX_RANK", + "MV2_COMM_WORLD_RANK", +) +ULTRANEST_WORKER_ENV_KEYS = ( + "EXOTIC_ULTRANEST_WORKERS", + "NEXTASTRO_EXOTIC_ULTRANEST_WORKERS", +) +ULTRANEST_WORKER_BACKEND_ENV = "EXOTIC_ULTRANEST_WORKER_BACKEND" +BYTES_PER_GIB = 1024 ** 3 +MIN_AUTO_POINTS_PER_WORKER = 8 +MEDIUM_AUTO_POINTS_PER_WORKER = 16 +HIGH_AUTO_POINTS_PER_WORKER = 24 +MAX_AUTO_POINTS_PER_WORKER = 32 +AUTO_POINTS_PER_WORKER_MULTIPLIER = 2 +AUTO_DRAW_RAM_FRACTION = 0.005 +MIN_AUTO_DRAW_RAM_BUDGET_BYTES = 64 * 1024 ** 2 +_PROCESS_LOGLIKE = None +_TK_CLEANUP_CLASSES = ("Image", "Variable") + + +def _is_enabled(value): + return str(value).strip().lower() in _TRUTHY + + +def _is_disabled(value): + return str(value).strip().lower() in _FALSEY + + +def _is_auto_worker_count(value): + return str(value).strip().lower() in _AUTO_WORKER_VALUES + + +def _coerce_positive_int(value, default=None): + try: + parsed = int(float(str(value).strip())) + except (TypeError, ValueError): + return default + + if parsed <= 0: + return default + return parsed + + +def _coerce_int(value, default=None): + try: + return int(float(str(value).strip())) + except (TypeError, ValueError): + return default + + +def _configured_mpi_int(env_keys, default=None): + for env_key in env_keys: + value = _coerce_int(os.environ.get(env_key), default=None) + if value is not None: + return value + return default + + +def get_mpi_status(): + """Return basic MPI status without requiring MPI to be installed.""" + env_size = _configured_mpi_int(MPI_SIZE_ENV_KEYS, default=1) + env_rank = _configured_mpi_int(MPI_RANK_ENV_KEYS, default=0) + try: + from mpi4py import MPI + + comm = MPI.COMM_WORLD + return { + "available": True, + "size": int(comm.Get_size()), + "rank": int(comm.Get_rank()), + "source": "mpi4py", + "error": None, + } + except Exception as exc: + return { + "available": False, + "size": max(int(env_size or 1), 1), + "rank": max(int(env_rank or 0), 0), + "source": "environment", + "error": str(exc), + } + + +def is_mpi_worker_process(): + status = get_mpi_status() + return status["size"] > 1 and status["rank"] > 0 + + +def _is_colab_runtime(): + return bool( + os.environ.get("COLAB_RELEASE_TAG") + or os.environ.get("GOOGLE_COLAB") + or "google.colab" in sys.modules + ) + + +def _available_cpu_count(): + process_cpu_count = getattr(os, "process_cpu_count", None) + if callable(process_cpu_count): + count = process_cpu_count() + else: + count = os.cpu_count() + return max(_coerce_positive_int(count, default=1), 1) + + +def _configured_ultranest_workers(): + for env_key in ULTRANEST_WORKER_ENV_KEYS: + value = os.environ.get(env_key) + if value in (None, ""): + continue + if _is_disabled(value): + return 1 + if _is_auto_worker_count(value): + return _available_cpu_count() + parsed = _coerce_positive_int(value, default=None) + if parsed is not None: + return parsed + + return _available_cpu_count() + + +def _configured_ultranest_worker_backend(): + backend = str(os.environ.get(ULTRANEST_WORKER_BACKEND_ENV, "")).strip().lower() + if backend in {"process", "processes", "multiprocessing"}: + return "process" + if backend in {"thread", "threads", "threading"}: + return "thread" + return "none" + + +def _system_total_memory_bytes(): + if hasattr(os, "sysconf"): + try: + page_size = int(os.sysconf("SC_PAGE_SIZE")) + page_count = int(os.sysconf("SC_PHYS_PAGES")) + if page_size > 0 and page_count > 0: + return page_size * page_count + except (AttributeError, OSError, TypeError, ValueError): + pass + + if sys.platform.startswith("win"): + try: + import ctypes + + class MEMORYSTATUSEX(ctypes.Structure): + _fields_ = [ + ("dwLength", ctypes.c_ulong), + ("dwMemoryLoad", ctypes.c_ulong), + ("ullTotalPhys", ctypes.c_ulonglong), + ("ullAvailPhys", ctypes.c_ulonglong), + ("ullTotalPageFile", ctypes.c_ulonglong), + ("ullAvailPageFile", ctypes.c_ulonglong), + ("ullTotalVirtual", ctypes.c_ulonglong), + ("ullAvailVirtual", ctypes.c_ulonglong), + ("sullAvailExtendedVirtual", ctypes.c_ulonglong), + ] + + memory_status = MEMORYSTATUSEX() + memory_status.dwLength = ctypes.sizeof(MEMORYSTATUSEX) + if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(memory_status)): + return int(memory_status.ullTotalPhys) + except Exception: + pass + + return None + + +def _auto_points_per_worker(cpu_count, total_memory_bytes): + if total_memory_bytes is None: + return MEDIUM_AUTO_POINTS_PER_WORKER * AUTO_POINTS_PER_WORKER_MULTIPLIER + + ram_per_cpu_gib = total_memory_bytes / max(int(cpu_count or 1), 1) / BYTES_PER_GIB + if ram_per_cpu_gib >= 2.0: + return MAX_AUTO_POINTS_PER_WORKER * AUTO_POINTS_PER_WORKER_MULTIPLIER + if ram_per_cpu_gib >= 1.0: + return HIGH_AUTO_POINTS_PER_WORKER * AUTO_POINTS_PER_WORKER_MULTIPLIER + if ram_per_cpu_gib >= 0.5: + return MEDIUM_AUTO_POINTS_PER_WORKER * AUTO_POINTS_PER_WORKER_MULTIPLIER + return MIN_AUTO_POINTS_PER_WORKER * AUTO_POINTS_PER_WORKER_MULTIPLIER + + +def _estimate_ultranest_draw_point_bytes(sampler): + x_dim = _coerce_positive_int(getattr(sampler, "x_dim", None), default=None) + num_params = _coerce_positive_int(getattr(sampler, "num_params", None), default=None) + coordinate_count = 16 + if x_dim is not None and num_params is not None: + coordinate_count = max(16, 3 + x_dim + num_params) + + return max(1024, coordinate_count * np.dtype(float).itemsize * 16) + + +def _apply_auto_ultranest_draw_sizes(sampler, worker_count): + worker_count = max(int(worker_count or 1), 1) + if worker_count <= 1 or getattr(sampler, "draw_multiple", True) is False: + return None + + current_min = _coerce_positive_int(getattr(sampler, "ndraw_min", None), default=None) + current_max = _coerce_positive_int(getattr(sampler, "ndraw_max", None), default=None) + if current_min is None and current_max is None: + return None + if current_min is None: + current_min = 128 + if current_max is None: + current_max = max(current_min, 65536) + if current_max < current_min: + current_max = current_min + + total_memory_bytes = _system_total_memory_bytes() + cpu_count = _available_cpu_count() + points_per_worker = _auto_points_per_worker(cpu_count, total_memory_bytes) + target_min = max(current_min, worker_count * points_per_worker) + memory_limited_max = current_max + + if total_memory_bytes is not None: + memory_budget = max( + int(total_memory_bytes * AUTO_DRAW_RAM_FRACTION), + MIN_AUTO_DRAW_RAM_BUDGET_BYTES, + ) + point_bytes = _estimate_ultranest_draw_point_bytes(sampler) + memory_limited_max = max(current_min, min(current_max, memory_budget // point_bytes)) + target_min = min(target_min, memory_limited_max) + + target_min = int(max(current_min, min(current_max, target_min))) + target_max = int(max(target_min, min(current_max, memory_limited_max))) + if target_min == current_min and target_max == current_max: + return None + + sampler.ndraw_min = target_min + sampler.ndraw_max = target_max + + ram_gib = None + if total_memory_bytes is not None: + ram_gib = total_memory_bytes / BYTES_PER_GIB + + return { + "ndraw_min": target_min, + "ndraw_max": target_max, + "workers": worker_count, + "cpu_count": cpu_count, + "ram_gib": ram_gib, + "points_per_worker": points_per_worker, + } + + +def _process_loglike_chunk(chunk): + if _PROCESS_LOGLIKE is None: + raise RuntimeError("UltraNest process worker was not initialized.") + return _PROCESS_LOGLIKE(chunk) + + +def _noop_tk_destructor(_instance): + return None + + +def _suppress_inherited_tk_cleanup(): + """Avoid noisy Tk destructor calls in forked worker processes.""" + tkinter_module = sys.modules.get("tkinter") + if tkinter_module is None: + return False + + patched = False + for class_name in _TK_CLEANUP_CLASSES: + tk_class = getattr(tkinter_module, class_name, None) + if tk_class is None or getattr(tk_class, "_exotic_worker_tk_cleanup_suppressed", False): + continue + + try: + original_del = getattr(tk_class, "__del__", None) + if original_del is None: + continue + setattr(tk_class, "_exotic_worker_original_del", original_del) + setattr(tk_class, "__del__", _noop_tk_destructor) + setattr(tk_class, "_exotic_worker_tk_cleanup_suppressed", True) + patched = True + except Exception: + continue + + return patched + + +def suppress_inherited_tk_cleanup_in_worker(): + return _suppress_inherited_tk_cleanup() + + +@contextmanager +def suppress_tk_cleanup_during_process_pool(): + suppress_inherited_tk_cleanup_in_worker() + restore_gc_after_pool = gc.isenabled() + if restore_gc_after_pool: + gc.disable() + try: + yield + finally: + if restore_gc_after_pool: + gc.enable() + + +@contextmanager +def _parallel_vectorized_loglike(sampler, workers=None): + worker_count = max(int(workers or _configured_ultranest_workers()), 1) + backend = _configured_ultranest_worker_backend() + status = get_mpi_status() + if int(status.get("size") or 1) > 1: + worker_count = 1 + + original_loglike = getattr(sampler, "loglike", None) + if worker_count <= 1 or backend == "none" or not callable(original_loglike): + yield 1, "single", None + return + + pool = None + executor = None + process_pool_guard = None + if backend == "process": + if not sys.platform.startswith("linux") or _is_colab_runtime(): + yield 1, "single", None + return + global _PROCESS_LOGLIKE + process_pool_guard = suppress_tk_cleanup_during_process_pool() + process_pool_guard.__enter__() + _PROCESS_LOGLIKE = original_loglike + try: + ctx = multiprocessing.get_context("fork") + pool = ctx.Pool(processes=worker_count, initializer=suppress_inherited_tk_cleanup_in_worker) + except Exception: + _PROCESS_LOGLIKE = None + process_pool_guard.__exit__(*sys.exc_info()) + raise + elif backend == "thread": + executor = ThreadPoolExecutor(max_workers=worker_count, thread_name_prefix="exotic-ultranest") + else: + yield 1, "single", None + return + + def parallel_loglike(params): + params_array = np.asarray(params) + if params_array.ndim != 2 or params_array.shape[0] < 2: + return original_loglike(params) + + chunk_count = min(worker_count, params_array.shape[0]) + chunks = [chunk for chunk in np.array_split(params_array, chunk_count) if len(chunk)] + if pool is not None: + results = pool.map(_process_loglike_chunk, chunks) + else: + results = list(executor.map(original_loglike, chunks)) + return np.concatenate([np.atleast_1d(result) for result in results]) + + sampler.loglike = parallel_loglike + draw_sizes = _apply_auto_ultranest_draw_sizes(sampler, worker_count) + try: + yield worker_count, backend, draw_sizes + finally: + sampler.loglike = original_loglike + if pool is not None: + try: + pool.close() + pool.join() + finally: + _PROCESS_LOGLIKE = None + if process_pool_guard is not None: + process_pool_guard.__exit__(None, None, None) + if executor is not None: + executor.shutdown(wait=True) + + +def _configured_min_num_live_points(default=DEFAULT_MIN_NUM_LIVE_POINTS): + for env_key in MIN_LIVE_POINTS_ENV_KEYS: + value = _coerce_positive_int(os.environ.get(env_key), default=None) + if value is not None: + return value + return default + + +def _apply_default_run_kwargs(run_kwargs): + kwargs = dict(DEFAULT_RUN_KWARGS) + kwargs["min_num_live_points"] = _configured_min_num_live_points() + kwargs.update({} if run_kwargs is None else dict(run_kwargs)) + return kwargs + + +def supports_ultranest_live_status(stream=None): + """Return True only when rich UltraNest status is explicitly enabled.""" + if _is_enabled(os.environ.get("EXOTIC_ULTRANEST_PLAIN_PROGRESS", "")): + return False + if not _is_enabled(os.environ.get("EXOTIC_ULTRANEST_RICH_PROGRESS", "")): + return False + + if stream is None: + stream = sys.stdout + if stream is None: + return False + + isatty = getattr(stream, "isatty", None) + if not callable(isatty) or not isatty(): + return False + + if _is_enabled(os.environ.get("CI", "")): + return False + + term = str(os.environ.get("TERM", "")).strip().lower() + if term == "dumb": + return False + + return True + + +def supports_ultranest_simple_status(): + """Return True only when simple UltraNest status is explicitly enabled.""" + return _is_enabled(os.environ.get("EXOTIC_ULTRANEST_PLAIN_PROGRESS", "")) + + +def _progress_mode(verbose, stream=None): + if not verbose: + return "silent" + if supports_ultranest_live_status(stream=stream): + return "rich" + if supports_ultranest_simple_status(): + return "simple" + return "simple" + + +@contextmanager +def _mute_ultranest_logging(sampler): + muted = [] + seen = set() + + sampler_logger = getattr(sampler, "logger", None) + if isinstance(sampler_logger, logging.Logger): + muted.append(sampler_logger) + seen.add(id(sampler_logger)) + + for logger in logging.root.manager.loggerDict.values(): + if not isinstance(logger, logging.Logger): + continue + if not logger.name.startswith("ultranest"): + continue + if id(logger) in seen: + continue + muted.append(logger) + seen.add(id(logger)) + + states = [(logger, logger.disabled) for logger in muted] + try: + for logger in muted: + logger.disabled = True + yield + finally: + for logger, disabled in states: + logger.disabled = disabled + + +def _traceback_mentions_ultranest_mlfriends(exc, function_name=None): + traceback = exc.__traceback__ + while traceback is not None: + filename = str(traceback.tb_frame.f_code.co_filename).replace("\\", "/") + frame_function = str(traceback.tb_frame.f_code.co_name).rsplit(".", 1)[-1] + if ( + "ultranest/mlfriends" in filename + and (function_name is None or frame_function == function_name) + ): + return True + traceback = traceback.tb_next + return False + + +def _is_ultranest_degenerate_region_error(exc): + if isinstance(exc, AssertionError): + # UltraNest 4.5.0 asserts here when a bootstrap region contains one + # unique point and its covariance is therefore non-finite. + return _traceback_mentions_ultranest_mlfriends(exc, "bounding_ellipsoid") + + if not isinstance(exc, ValueError): + return False + + message = str(exc) + if "Buffer has wrong number of dimensions" not in message: + return False + if "expected 2" not in message or "got 0" not in message: + return False + return _traceback_mentions_ultranest_mlfriends(exc) + + +def _run_sampler_with_degenerate_region_guard(sampler, kwargs): + try: + return sampler.run(**kwargs) + except (AssertionError, ValueError) as exc: + if not _is_ultranest_degenerate_region_error(exc): + raise + raise np.linalg.LinAlgError( + "UltraNest failed while building a degenerate sampling region." + ) from exc + + +def _read_float(mapping, *keys): + for key in keys: + if key not in mapping: + continue + try: + return float(mapping[key]) + except (TypeError, ValueError): + continue + return None + + +def _read_int(mapping, *keys): + for key in keys: + if key not in mapping: + continue + try: + return int(mapping[key]) + except (TypeError, ValueError): + continue + return None + + +def _extract_info(args, kwargs): + info = kwargs.get("info") + if isinstance(info, dict): + return info + for item in reversed(args): + if isinstance(item, dict): + return item + return {} + + +class _UltraNestSimpleProgress: + def __init__(self, stream=None, interval_seconds=DEFAULT_PROGRESS_INTERVAL_SECONDS, bar_width=28): + self.stream = stream if stream is not None else sys.stdout + self.interval_seconds = max(float(interval_seconds), 0.0) + self.bar_width = max(int(bar_width), 8) + self.start_time = time.monotonic() + self.last_emit = self.start_time - self.interval_seconds + self.iteration = None + self.evaluations = None + self.progress = 0.0 + + def _write(self, message): + if self.stream is None: + return + self.stream.write(message + "\n") + self.stream.flush() + + @staticmethod + def _progress_from_info(info): + progress = _read_float(info, "progress", "fraction_done") + if progress is not None: + if progress > 1.0: + progress /= 100.0 + return min(max(progress, 0.0), 1.0) + + remainder = _read_float(info, "remainder_fraction", "remaining_fraction", "frac_remain") + if remainder is not None: + if remainder > 1.0: + remainder /= 100.0 + return min(max(1.0 - remainder, 0.0), 1.0) + + logz = _read_float(info, "logz") + logz_remain = _read_float(info, "logz_remain", "logzremain") + if logz is None or logz_remain is None or not math.isfinite(logz_remain): + return None + if not math.isfinite(logz): + return 0.0 + + delta = logz_remain - logz + if delta >= 50: + return 0.0 + if delta <= -50: + return 1.0 + return 1.0 / (1.0 + math.exp(delta)) + + def _elapsed(self): + total = max(int(time.monotonic() - self.start_time), 0) + hours, rem = divmod(total, 3600) + minutes, seconds = divmod(rem, 60) + if hours: + return f"{hours:d}:{minutes:02d}:{seconds:02d}" + return f"{minutes:02d}:{seconds:02d}" + + def _line(self, done=False): + fraction = 1.0 if done else min(max(self.progress, 0.0), 1.0) + filled = int(round(fraction * self.bar_width)) + bar = "#" * filled + "-" * (self.bar_width - filled) + pct = f"{100.0 * fraction:6.2f}%" + iteration = "?" if self.iteration is None else str(self.iteration) + evaluations = "?" if self.evaluations is None else str(self.evaluations) + state = "done" if done else "running" + return ( + f"[ultranest] {state} {pct} [{bar}] " + f"it={iteration} evals={evaluations} elapsed={self._elapsed()}" + ) + + def start(self): + heartbeat = f"{self.interval_seconds:g}s" + self._write(f"[ultranest] Using simple progress updates ({heartbeat} heartbeat).") + + def update(self, *args, **kwargs): + info = _extract_info(args, kwargs) + if not info: + return + + progress = self._progress_from_info(info) + if progress is not None: + self.progress = progress + + iteration = _read_int(info, "it", "iteration") + if iteration is not None: + self.iteration = iteration + + evaluations = _read_int(info, "ncall", "ncalls", "evals") + if evaluations is not None: + self.evaluations = evaluations + + def maybe_emit(self, force=False): + now = time.monotonic() + if not force and (now - self.last_emit) < self.interval_seconds: + return + self.last_emit = now + self._write(self._line(done=False)) + + def finish(self): + self._write(self._line(done=True)) + + +def run_reactive_sampler( + sampler, + run_kwargs=None, + verbose=True, + stream=None, + interval_seconds=DEFAULT_PROGRESS_INTERVAL_SECONDS, +): + """ + Run an UltraNest sampler with simple text progress by default. + + Set EXOTIC_ULTRANEST_RICH_PROGRESS=1 for UltraNest's native status output. + Use verbose=False to silence all progress updates. + """ + kwargs = _apply_default_run_kwargs(run_kwargs) + mode = "silent" if is_mpi_worker_process() else _progress_mode(verbose=verbose, stream=stream) + output_stream = stream if stream is not None else sys.stdout + + with _parallel_vectorized_loglike(sampler) as (worker_count, worker_backend, draw_sizes): + if draw_sizes is not None and mode != "silent": + memory_label = "unknown RAM" + if draw_sizes["ram_gib"] is not None: + memory_label = f"{draw_sizes['ram_gib']:.1f} GiB RAM" + print( + "[ultranest] Auto proposal draw sizes: " + f"ndraw_min={draw_sizes['ndraw_min']}, ndraw_max={draw_sizes['ndraw_max']} " + f"({draw_sizes['workers']} workers, {draw_sizes['points_per_worker']} points/worker, " + f"{memory_label}).", + file=output_stream, + flush=True, + ) + if worker_count > 1 and mode != "silent": + worker_label = "processes" if worker_backend == "process" else "threads" + print( + f"[ultranest] Using {worker_count} worker {worker_label} for vectorized likelihood batches.", + file=output_stream, + flush=True, + ) + + if mode == "silent": + kwargs["show_status"] = False + kwargs["viz_callback"] = False + with _mute_ultranest_logging(sampler): + return _run_sampler_with_degenerate_region_guard(sampler, kwargs) + + if mode == "rich": + kwargs.setdefault("show_status", True) + return _run_sampler_with_degenerate_region_guard(sampler, kwargs) + + progress = _UltraNestSimpleProgress(stream=stream, interval_seconds=interval_seconds) + upstream_callback = kwargs.get("viz_callback") + + def callback(*args, **callback_kwargs): + progress.update(*args, **callback_kwargs) + progress.maybe_emit(force=False) + if callable(upstream_callback): + upstream_callback(*args, **callback_kwargs) + + kwargs["show_status"] = False + kwargs["viz_callback"] = callback + + progress.start() + try: + with _mute_ultranest_logging(sampler): + result = _run_sampler_with_degenerate_region_guard(sampler, kwargs) + finally: + progress.finish() + return result diff --git a/exotic/exotic.py b/exotic/exotic.py index f7001f9d..71d17444 100644 --- a/exotic/exotic.py +++ b/exotic/exotic.py @@ -57,8 +57,24 @@ # standard imports import argparse +import csv +import copy +from datetime import datetime +import faulthandler +from functools import lru_cache +import inspect +import json import hashlib -from time import sleep +from math import atan2, cos, radians, sin, sqrt +import multiprocessing +import os +import shutil +import sys +import threading +import traceback +from concurrent.futures import ProcessPoolExecutor as _ProcessPoolExecutor, ThreadPoolExecutor, as_completed +from time import sleep, perf_counter +from types import SimpleNamespace # Image alignment import import astroalign as aa aa.PIXEL_TOL = 1 @@ -68,80 +84,156 @@ from astropy.coordinates import SkyCoord, EarthLocation, AltAz from astropy.io import fits from astropy.time import Time +from astropy.timeseries import BoxLeastSquares from astropy.visualization import astropy_mpl_style from astropy.wcs import WCS, FITSFixedWarning -from astroquery.simbad import Simbad -from astroquery.gaia import Gaia # UTC to BJD converter import from barycorrpy.utc_tdb import JDUTC_to_BJDTDB -import copy # julian conversion imports import dateutil.parser as dup import imreg_dft as ird from pathlib import Path -import pyvo as vo import logging -from logging.handlers import TimedRotatingFileHandler +import tempfile from matplotlib.animation import FuncAnimation # Pyplot imports +import bottleneck as bn import matplotlib.pyplot as plt import numpy as np # photometry -from photutils.aperture import CircularAperture -import pandas as pd +from photutils.aperture import CircularAperture, CircularAnnulus import re import requests # scipy imports from scipy.optimize import least_squares -from scipy.stats import mode from scipy.signal import savgol_filter -from scipy.ndimage import binary_erosion -from skimage.util import view_as_windows +from scipy.ndimage import binary_erosion, gaussian_filter, label as ndimage_label, maximum_filter, median_filter +from scipy.special import ndtri +from skimage.registration import phase_cross_correlation from skimage.transform import SimilarityTransform -from skimage.color import rgb2gray # error handling for scraper -from tenacity import retry, stop_after_delay +from tenacity import RetryError, retry, retry_if_exception, stop_after_attempt, stop_after_delay, wait_fixed # color, color_demosaicing from colour_demosaicing import demosaicing_CFA_Bayer_bilinear # ########## EXOTIC imports ########## try: # light curve numerics - from .api.elca import lc_fitter, binner, transit, get_phase + from .api.elca import lc_fitter, transit, get_phase except ImportError: # package import - from api.elca import lc_fitter, binner, transit, get_phase + from api.elca import lc_fitter, transit, get_phase try: # output files - from inputs import Inputs, comparison_star_coords + from inputs import Inputs, NEXTASTRO_GAIA_DISTPM_ENDPOINT, comparison_star_coords except ImportError: # package import - from .inputs import Inputs, comparison_star_coords + from .inputs import Inputs, NEXTASTRO_GAIA_DISTPM_ENDPOINT, comparison_star_coords try: # ld from .api.ld import LimbDarkening, ld_re_punct_p except ImportError: # package import from api.ld import LimbDarkening, ld_re_punct_p try: # plate solution - from .api.plate_solution import PlateSolution + from .api.plate_solution import NextAstroPlateSolution, PlateSolution except ImportError: # package import - from api.plate_solution import PlateSolution + from api.plate_solution import NextAstroPlateSolution, PlateSolution +try: + from .api.ultranest_utils import ( + get_mpi_status, + suppress_inherited_tk_cleanup_in_worker, + suppress_tk_cleanup_during_process_pool, + ) +except ImportError: + from api.ultranest_utils import ( + get_mpi_status, + suppress_inherited_tk_cleanup_in_worker, + suppress_tk_cleanup_during_process_pool, + ) +try: + from .api.http_compression import build_compressed_json_request +except ImportError: + from api.http_compression import build_compressed_json_request try: # nea from .api.nea import NASAExoplanetArchive except ImportError: # package import from api.nea import NASAExoplanetArchive try: # output files - from output_files import OutputFiles, AIDOutputFiles + from output_files import ( + OutputFiles, + AIDOutputFiles, + baseline_fixed_after_detrending, + pre_detrending_baseline_report, + empirical_red_noise_error_scale, + differential_magnitude_series_from_fit, + fit_empirical_transit_uncertainty, + fit_impact_parameter_value_error, + fit_parameter_model_data_uncertainty, + format_parameter_with_error, + formatted_transit_depth_parameters, + save_comp_star_calibration_summary, + write_differential_magnitude_csv, + ) except ImportError: # package import - from .output_files import OutputFiles, AIDOutputFiles + from .output_files import ( + OutputFiles, + AIDOutputFiles, + baseline_fixed_after_detrending, + pre_detrending_baseline_report, + empirical_red_noise_error_scale, + differential_magnitude_series_from_fit, + fit_empirical_transit_uncertainty, + fit_impact_parameter_value_error, + fit_parameter_model_data_uncertainty, + format_parameter_with_error, + formatted_transit_depth_parameters, + save_comp_star_calibration_summary, + write_differential_magnitude_csv, + ) +try: + from transit_depth import fit_transit_depth_summary +except ImportError: + from .transit_depth import fit_transit_depth_summary try: from plate_status import PlateStatus except ImportError: from .plate_status import PlateStatus try: # plots from plots import plot_fov, plot_centroids, plot_obs_stats, plot_final_lightcurve, plot_flux, \ - plot_stellar_variability, plot_variable_residuals + plot_prior_posterior_comparison, plot_ktmf_qc_metrics, \ + plot_stellar_variability, plot_differential_magnitude, plot_variable_residuals, plot_comp_star_pairwise_matrix, \ + plot_comp_star_calibration_series, plot_individual_comp_star_calibration_series, \ + plot_comp_star_candidate_lightcurve_fits, plot_comp_star_suitability, \ + plot_adaptive_aperture_diagnostics except ImportError: # package import from .plots import plot_fov, plot_centroids, plot_obs_stats, plot_final_lightcurve, plot_flux, \ - plot_stellar_variability, plot_variable_residuals + plot_prior_posterior_comparison, plot_ktmf_qc_metrics, \ + plot_stellar_variability, plot_differential_magnitude, plot_variable_residuals, plot_comp_star_pairwise_matrix, \ + plot_comp_star_calibration_series, plot_individual_comp_star_calibration_series, \ + plot_comp_star_candidate_lightcurve_fits, plot_comp_star_suitability, \ + plot_adaptive_aperture_diagnostics try: # tools - from utils import round_to_2, user_input + from utils import ( + AAVSO_OUTPUT_FOLDER_NAME, + MAX_APPARENT_MAGNITUDE, + coerce_boolean_config_value, + filename_date_token, + format_value_with_uncertainty, + is_usable_apparent_magnitude, + magnitude_text, + normalized_magnitude_error, + round_to_2, + safe_output_filename, + user_input, + ) except ImportError: # package import - from .utils import round_to_2, user_input + from .utils import ( + AAVSO_OUTPUT_FOLDER_NAME, + MAX_APPARENT_MAGNITUDE, + coerce_boolean_config_value, + filename_date_token, + format_value_with_uncertainty, + is_usable_apparent_magnitude, + magnitude_text, + normalized_magnitude_error, + round_to_2, + safe_output_filename, + user_input, + ) try: # simple version from .version import __version__ except ImportError: # package import @@ -158,1629 +250,31724 @@ # logging -- https://docs.python.org/3/library/logging.html log = logging.getLogger(__name__) +_RUNTIME_LOGGING_CONFIGURED = False +_EXCEPTION_HOOKS_INSTALLED = False +_UNHANDLED_EXCEPTION_LOGGED = False +_BJD_FALLBACK_WARNING_LOGGED = False +_RUNTIME_FILE_HANDLER_NAME = "exotic-runtime-file" +_RUNTIME_CONSOLE_HANDLER_NAME = "exotic-runtime-console" +_RUNTIME_LOG_BASENAME = None +_RUNTIME_LOG_PATH = None +_RUNTIME_TRACEBACK_WATCHDOG_SECONDS_ENV = "EXOTIC_RUNTIME_TRACEBACK_WATCHDOG_SECONDS" +_RUNTIME_TRACEBACK_WATCHDOG_DEFAULT_SECONDS = 1800.0 +_RUNTIME_TRACEBACK_WATCHDOG_ACTIVE = False +_mid_transit_warning_reported = False +RELATIVE_FLUX_MAX = 2.0 # Legacy threshold retained for compatibility; no longer used as a hard rejection cap. +AIRMASS_FLAT_RANGE_THRESHOLD = 0.05 +LIGHTCURVE_MIN_VALID_POINTS = 5 +STELLAR_VARIABILITY_ONLY_DEFAULT = False +STELLAR_VARIABILITY_ENSEMBLE_DEFAULT = True +REQUIRE_APPARENT_MAGNITUDES_DEFAULT = True +USE_EXACTLY_PROVIDED_COMPARISONS_DEFAULT = False +STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS = 2 +TRANSIT_ENSEMBLE_MAX_COMPARISONS_DEFAULT = 5 +STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS = 5 +STELLAR_VARIABILITY_APERTURE_ESTIMATION_MAX_COMPARISONS = 5 +STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_SIGMA = 3.0 +STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_FLOOR = 1.0e-4 +STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_FLOOR_FRACTION = 0.05 +STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_HIGH_THRESHOLD_FLOOR_MAG = 0.01 +STELLAR_VARIABILITY_COMPARISON_GAP_MIN_RATIO = 5.0 +STELLAR_VARIABILITY_COMPARISON_GAP_MIN_SECONDS = 30.0 +STELLAR_VARIABILITY_COMPARISON_GAP_WINDOW_FRAMES = 50 +STELLAR_VARIABILITY_COMPARISON_GAP_MIN_POINTS = 10 +STELLAR_VARIABILITY_COMPARISON_GAP_MAX_STEP_MAG = 0.02 +STELLAR_VARIABILITY_COMPARISON_GAP_MIN_SIGNIFICANCE = 5.0 +PHOTOMETER_FORTUITOUS_VARIABLES_DEFAULT = True +USE_SINGLE_COMPARISON_FOR_FORTUITOUS_VARIABLES_DEFAULT = True +USE_NEXTASTRO_VSX_CACHE_FIRST_DEFAULT = False +FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR = 0.05 +FORTUITOUS_VARIABLE_OPTIMAL_MAX_PERIOD_DAYS = 10.0 +FORTUITOUS_VARIABLE_OPTIMAL_MIN_AMPLITUDE_MAG = 0.3 +FORTUITOUS_VARIABLE_VSX_MAGNITUDE_LIMIT = 20.0 +REJECT_OVEREXPOSED_STARS_DEFAULT = True +SATURATION_VALUE_DEFAULT = 65535.0 +OVEREXPOSURE_THRESHOLD_FRACTION_DEFAULT = 0.9 +MICROOBSERVATORY_TELESCOP_SATURATION_VALUES = { + 'cecilia': 4096.0, +} +COMPARISON_STAR_MIN_COVERAGE_FRACTION = 0.8 +COMPARISON_STAR_MIN_VALID_FRAMES = 5 +COMPARISON_STAR_COVERAGE_SIGMA = 3.0 # Legacy constant; coverage rejection is fraction-based. +COMPARISON_STAR_COVERAGE_MAX_ITERS = 10 +COMPARISON_STAR_SUITABILITY_OUTLIER_SIGMA = 4.25 +COMPARISON_STAR_SUITABILITY_MIN_CANDIDATES = 5 +COMPARISON_STAR_SUITABILITY_MAX_ITERS = 10 +COMPARISON_STAR_PRESCORE_SIGMA_CLIP = 3.0 +COMPARISON_STAR_PRESCORE_MAX_CLIP_ITERS = 3 +COMPARISON_STAR_PRESCORE_SCATTER_FLOOR = 1e-4 +PSF_FRAME_QUALITY_SIGMA = 4.0 +PSF_FRAME_QUALITY_MAX_CLIP_ITERS = 3 +PSF_FRAME_QUALITY_SEEING_MIN_FRACTIONAL_DEVIATION = 0.5 +PSF_FRAME_QUALITY_AMPLITUDE_MIN_FRACTIONAL_DEVIATION = 0.25 +PSF_TARGET_QUALITY_MAX_COMP_SIGMA_RATIO = 3.0 +COMPARISON_IMAGE_OUTLIER_SIGMA = COMPARISON_STAR_SUITABILITY_OUTLIER_SIGMA +COMPARISON_IMAGE_OUTLIER_MIN_ACTIVE_STARS = 3 +COMPARISON_IMAGE_OUTLIER_MIN_VALID_PAIRS = 2 +COMPARISON_IMAGE_OUTLIER_MIN_SCATTER = 1e-4 +COMPARISON_CANDIDATE_FRAME_OUTLIER_MIN_VALID_PAIRS = 2 +OUT_OF_TRANSIT_BASELINE_DEPTH_FRACTION = 0.05 +OUT_OF_TRANSIT_BASELINE_MIN_SIDE_POINTS_DEFAULT = 12 +FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT = 1.0 +ULTRANEST_MIN_NUM_LIVE_POINTS_DEFAULT = 200 +ULTRANEST_MIN_NUM_LIVE_POINTS_ENV = "EXOTIC_ULTRANEST_MIN_NUM_LIVE_POINTS" +FAST_ULTRANEST_BEFORE_FINAL_RUN_DEFAULT = True +FAST_ULTRANEST_MAX_BINNED_POINTS = 20 +FAST_ULTRANEST_MIN_POINTS_TO_BIN = 60 +LEGACY_PSF_FLUX_MODE_DEFAULT = False +FINAL_FIT_PHASE_RESIDUAL_CLIP_DEFAULT = True +FINAL_RESIDUAL_REJECTION_DEFAULT = True +FINAL_RESIDUAL_REJECTION_SIGMA = 3.0 +FINAL_RESIDUAL_REJECTION_MAX_CLIP_ITERS = 10 +FINAL_RESIDUAL_REJECTION_MAX_REFITS = 10 +NOISE_BUDGET_COMPONENT_KEYS = ( + 'source', + 'sky_aperture', + 'sky_estimate', + 'read', + 'dark', + 'flat', + 'scintillation', + 'total', +) +NOISE_BUDGET_SKY_MEDIAN_VARIANCE_FACTOR = np.pi / 2.0 +SCINTILLATION_COEFFICIENT_DEFAULT = 0.09 +PSF_EFFECTIVE_NOISE_AREA_FACTOR = 4.0 * np.pi +NOISE_GAIN_HEADER_KEYS = ('EGAIN', 'EPERADU', 'E_PER_ADU', 'GAIN_EAD', 'CCDGAIN', 'GAIN') +NOISE_READ_HEADER_KEYS = ('RDNOISE', 'READNOI', 'READNOIS', 'READNSE', 'RN_E', 'RON') +NOISE_DARK_HEADER_KEYS = ('DARKCUR', 'DARKCURR', 'DARKRATE', 'DCURR', 'DARK_EPS', 'PBDKCURR') +NOISE_FLAT_HEADER_KEYS = ('FLATERR', 'FLATFR', 'FLATFRAC', 'FFERR', 'FLATUNC') +NOISE_SCINTILLATION_HEADER_KEYS = ('SCINCOEF', 'SCINTC') +NOISE_APERTURE_HEADER_KEYS = ('TELAPER', 'APERTURE') +NOISE_APERTURE_MM_HEADER_KEYS = ('APR-DIA', 'APTDIA', 'APERTMM') +COMPARISON_PREFLIGHT_FIELD_SCORE_RELATIVE_BAND = 0.25 +COMPARISON_PREFLIGHT_FIELD_SCORE_ABSOLUTE_BAND = 2.5e-4 +PARTIAL_COVERAGE_RPRS_POSTERIOR_MAX_RETRIES = 1 +PARTIAL_COVERAGE_ARS_POSTERIOR_MAX_RETRIES = 0 +PARTIAL_COVERAGE_IMPACT_PARAMETER_POSTERIOR_MAX_RETRIES = 0 +PROMISING_PARTIAL_COMPARISON_KTMF_MIN = 3.0 +COMPARISON_SELECTION_MAX_SCATTER_MULTIPLIER = 1.5 +SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_DEFAULT = True +SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_ENV = "EXOTIC_SPARSE_POSTERIOR_LIVE_POINT_RETRY" +SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT = 5 +SPARSE_POSTERIOR_RETRY_PARAMETER_KEYS = ('rprs', 'tmid', 'ars') +SPARSE_POSTERIOR_MIN_EFFECTIVE_SAMPLES_FLOOR = 1000 +SPARSE_POSTERIOR_MIN_EFFECTIVE_SAMPLES_PER_LIVE_POINT = 5.0 +SPARSE_POSTERIOR_MIN_OCCUPIED_BINS = 8 +SPARSE_POSTERIOR_MIN_OCCUPIED_BIN_FRACTION = 0.65 +SPARSE_POSTERIOR_MIN_EFFECTIVE_SAMPLES_PER_OCCUPIED_BIN = 25.0 +RPRS_POSTERIOR_MAX_RETRIES_DEFAULT = 5 +RPRS_SEARCH_BOUND_MIN = 0.0 +RPRS_SEARCH_BOUND_MAX_DEFAULT = 0.5 +RPRS_SEARCH_BOUND_ABSOLUTE_MAX = 1.0 +RPRS_SEARCH_BOUND_MAX = RPRS_SEARCH_BOUND_MAX_DEFAULT +RPRS_RANGE_RESTRICTION_DEFAULT = True +RPRS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT = 10.0 +RPRS_RANGE_RESTRICTION_ENABLED = RPRS_RANGE_RESTRICTION_DEFAULT +RPRS_RANGE_RESTRICTION_PERCENTAGE = RPRS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT +RPRS_DATA_UNCERTAINTY_BOUND_SIGMA = 3.0 +RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR_DEFAULT = True +RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR = RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR_DEFAULT +RPRS_RETRY_MIN_HALF_WIDTH = 0.05 +INITIAL_RPRS_BOUND_LOWER_SCALE = 0.0 +INITIAL_RPRS_BOUND_UPPER_SCALE = 3.0 +ARS_SEARCH_BOUND_MIN = 1e-6 +ARS_SEARCH_BOUND_FALLBACK_MAX = 100.0 +ARS_RANGE_RESTRICTION_DEFAULT = True +ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT = 10.0 +ARS_RANGE_RESTRICTION_ENABLED = ARS_RANGE_RESTRICTION_DEFAULT +ARS_RANGE_RESTRICTION_PERCENTAGE = ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT +ARS_POSTERIOR_MAX_RETRIES_DEFAULT = 5 +TOI_TIC_ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT = 30.0 +TOI_TIC_ARS_POSTERIOR_MAX_RETRIES_DEFAULT = 8 +ARS_RETRY_MIN_HALF_WIDTH = 0.0 +IMPACT_PARAMETER_POSTERIOR_MAX_RETRIES_DEFAULT = 5 +INCLINATION_SEARCH_BOUND_MIN = 0.0 +INCLINATION_SEARCH_BOUND_MAX = 90.0 +INITIAL_ARS_BOUND_SIGMA_MULTIPLIER = 5.0 +INITIAL_ARS_BOUND_FALLBACK_RELATIVE_HALF_WIDTH = 0.25 +DURATION_PRIOR_MONTE_CARLO_SAMPLES = 256 +DURATION_PRIOR_MIN_VALID_MONTE_CARLO_SAMPLES = 64 +DURATION_PRIOR_MIN_RELATIVE_SIGMA = 0.05 +DURATION_PRIOR_FALLBACK_RELATIVE_SIGMA = 0.15 +FINAL_FIT_TMID_HALF_DURATION_MULTIPLIER = 0.5 +EEBLS_DURATION_GRID_SIZE = 15 +EEBLS_DURATION_MIN_FRACTION = 0.5 +EEBLS_DURATION_MAX_FRACTION = 1.75 +EEBLS_TMID_HALF_WIDTH_DURATION_MULTIPLIER = 1.5 +EEBLS_MIN_VALID_POINTS = 10 +EPHEMERIS_BRACKETED_TMID_HALF_WIDTH_DURATION_MULTIPLIER = 2.0 +COMPARISON_STAR_DUPLICATE_DISTANCE_PIXELS = 15.0 +ROBUST_FLUX_MIN_FRACTION_OF_MEDIAN = 0.02 +ROBUST_FLUX_MIN_POINTS = 20 +WCS_REFERENCE_GEOMETRY_TOLERANCE_PIXELS = 5.0 +WCS_MIN_GEOMETRY_MATCH_FRACTION = 0.5 +PSF_FIT_MAX_SEED_OFFSET_PIXELS = 6.0 +PSF_FIT_MAX_AXIS_RATIO = 4.0 +PSF_FIT_MAX_SIGMA_PIXELS = 8.0 +PSF_FIT_SELECTION_MARGIN = 0.35 +PSF_ALIGNMENT_TARGET_WIDTH_MAX_COMP_RATIO = 3.0 +PSF_ALIGNMENT_CANDIDATE_SELECTION_MARGIN = 0.20 +LEGACY_ALIGNMENT_MAX_DIMENSION = 1600 +TIME_REJECTION_RANGE_DISPLAY_LIMIT = 6 +TIME_REJECTION_GROUP_GAP_CADENCE_MULTIPLIER = 2.5 +NEXTASTRO_VARIABILITY_MAX_RETRY_ATTEMPTS = 5 +NEXTASTRO_VARIABILITY_RETRY_WAIT_SECONDS = 10 +NEXTASTRO_VARIABILITY_RETRYABLE_HTTP_STATUS_CODES = {408, 425, 429, 500, 502, 503, 504} +NEXTASTRO_PHOTOMETRY_API_URL = 'https://photometry.nextastro.org' +NEXTASTRO_VSX_QUERY_URL = f'{NEXTASTRO_PHOTOMETRY_API_URL}/vsx_query' +NEXTASTRO_PHOTOMETRY_SINGLE_OBJECT_URL = f'{NEXTASTRO_PHOTOMETRY_API_URL}/single_object' +NEXTASTRO_PHOTOMETRY_OBJECTS_QUERY_URL = f'{NEXTASTRO_PHOTOMETRY_API_URL}/objects_query' +NEXTASTRO_VSX_QUERY_LIMIT = 200000 +NEXTASTRO_PHOTOMETRY_COLUMNS = ( + 'id', 'source_id', 'ra', 'dec', + 'Bmag', 'err_Bmag', 'Vmag', 'err_Vmag', + 'umag', 'err_umag', 'g', 'dg', 'r', 'dr', 'i', 'di', 'z', 'dz', +) +NEXTASTRO_PHOTOMETRY_IDENTITY_COLUMNS = ('id', 'source_id', 'ra', 'dec') +NEXTASTRO_PHOTOMETRY_FIELD_PADDING_ARCSEC = 30.0 +NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC = 2.0 +NEXTASTRO_GAIA_COLOR_LOOKUP_TIMEOUT_SECONDS = 10 +NEXTASTRO_GAIA_COLOR_LOOKUP_MAX_PER_SELECTOR = 25 +CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX = 0.05 +CATALOG_BV_REFERENCE_MAGNITUDE_ERROR_FALLBACK_MAX = 0.10 +VSP_COMPARISON_MATCH_TOLERANCE_PIXELS = 3.0 +AAVSO_VSP_REQUEST_TIMEOUT_SECONDS = 30 +AAVSO_VSP_RETRY_DELAY_SECONDS = 60 +AAVSO_VSP_MAX_RETRIES = 5 +REFERENCE_FALLBACK_COMPARISON_LIMIT = 10 +REFERENCE_FALLBACK_DETECTION_MAX_STARS = 60 +REFERENCE_FALLBACK_DETECTION_MIN_SEP_PIXELS = 12 +REFERENCE_FALLBACK_DETECTION_APERTURE_RADIUS_PIXELS = 4 +REFERENCE_FALLBACK_DETECTION_MIN_AREA_PIXELS = 3 +REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS = 10.0 +REFERENCE_FALLBACK_MIN_COMP_TARGET_SEP_PIXELS = 50.0 +AUTOMATIC_CALIBRATION_SELECTOR_DEFAULT_COUNT = 10 +AUTOMATIC_CALIBRATION_SELECTOR_BRIGHTNESS_MIN_RATIO = 0.5 +AUTOMATIC_CALIBRATION_SELECTOR_BRIGHTNESS_MAX_RATIO = 2.0 +AUTOMATIC_CALIBRATION_SELECTOR_MAX_DETECTIONS = 1000 +AUTOMATIC_CALIBRATION_SELECTOR_DETECTION_PERCENTILE = 98.0 +AUTOMATIC_CALIBRATION_SELECTOR_COLOR_MATCH_RADIUS_ARCSEC = 20.0 +BAD_PIXEL_DETECTION_FRACTION = 0.30 +BAD_PIXEL_PRECHECK_MIN_FRAMES = 5 +BAD_PIXEL_PROGRESS_LOG_INTERVAL = 25 +BAD_PIXEL_OUTLIER_SIGMA = 8.0 +BAD_PIXEL_GLOBAL_SIGMA = 3.0 +BAD_PIXEL_ISOLATION_SIGMA = 5.0 +BAD_PIXEL_ISOLATION_RATIO = 2.0 +MAX_MULTIPROCESS_BAD_PIXEL_WORKERS = 8 +BAD_PIXEL_COUNTS_FILENAME = "BadPixelDetectionCounts.fits" +BAD_PIXEL_MASK_FILENAME = "BadPixelMask.fits" +BAD_PIXEL_NEIGHBOR_FOOTPRINT = np.array( + [[1, 1, 1], + [1, 0, 1], + [1, 1, 1]], + dtype=bool, +) +TRANSIT_QC_DELTA_BIC_FAIL_THRESHOLD = 6.0 +TRANSIT_QC_DELTA_BIC_PASS_THRESHOLD = 10.0 +TRANSIT_QC_KTMF_FAIL_THRESHOLD = 3.0 +TRANSIT_QC_KTMF_PASS_THRESHOLD = 4.0 +TRANSIT_QC_MIN_EEBLS_SNR = 4.0 +TRANSIT_QC_DURATION_RATIO_MIN = 0.5 +TRANSIT_QC_DURATION_RATIO_MAX = 2.0 +TRANSIT_QC_DEFAULT_A2_BOUNDS = (-3.0, 3.0) +TRANSIT_QC_USE_DEVIATION_FROM_EXPECTED_DEFAULT = True +TRANSIT_QC_DEVIATION_SIGMA_DEFAULT = 5.0 +TRANSIT_QC_RPRS_DEVIATION_SYSTEMATIC_FLOOR_FRACTION = 0.05 +TRANSIT_QC_TMID_GAUSSIANITY_MIN_EFFECTIVE_SAMPLES = 200 +TRANSIT_QC_TMID_GAUSSIANITY_BOOTSTRAP_DRAWS = 48 +TRANSIT_QC_TMID_GAUSSIANITY_BOOTSTRAP_MAX_SAMPLES = 2000 +TRANSIT_QC_KTMF_COMPONENT_MAX_POINTS = { + 'deviation_from_expected_value': 2.0, + 'residual_scatter': 0.7, + 'residual_flatness': 1.0, + 'tmid_gaussianity': 1.0, + 'duration_consistency': 0.75, + 'eebls_depth_snr': 1.3, + 'sampling': 0.7, +} + + +def airmass_span(airmass): + values = np.asarray(airmass, dtype=float) + finite = values[np.isfinite(values)] + if finite.size == 0: + return np.nan + return float(np.nanmax(finite) - np.nanmin(finite)) + + +def should_skip_airmass_fit(airmass, max_span=AIRMASS_FLAT_RANGE_THRESHOLD): + span = airmass_span(airmass) + return np.isfinite(span) and span <= max_span + + +def annotate_airmass_fit(fit, airmass, skipped, max_span=AIRMASS_FLAT_RANGE_THRESHOLD, note=None): + if fit is None: + return + + span = airmass_span(airmass) + fit.airmass_span = span + fit.airmass_fit_threshold = max_span + fit.airmass_fit_skipped = bool(skipped) + fit.airmass_correction_note = None + if fit.airmass_fit_skipped: + if note: + fit.airmass_correction_note = note + elif np.isfinite(span): + fit.airmass_correction_note = ( + f"Skipped (airmass span {span:.4f} <= {max_span:.2f}); no airmass correction applied." + ) + else: + fit.airmass_correction_note = "Skipped; no airmass correction applied." + + +def annotate_out_of_transit_baseline_detrending( + fit, + applied, + note=None, + slope=None, + intercept=None, + reference_time_bjd_tdb=None, + pre_points=0, + post_points=0, +): + if fit is None: + return + + fit.oot_baseline_detrending_applied = bool(applied) + fit.oot_baseline_detrending_note = note + fit.oot_baseline_slope = slope + fit.oot_baseline_intercept = intercept + fit.oot_baseline_reference_time_bjd_tdb = reference_time_bjd_tdb + fit.oot_baseline_pre_points = int(pre_points) if pre_points is not None else 0 + fit.oot_baseline_post_points = int(post_points) if post_points is not None else 0 + + +def annotate_out_of_transit_baseline_parameter_fit( + fit, + applied, + note=None, + pre_points=0, + post_points=0, + a0=None, + a0_error=None, + a2=None, + a2_error=None, +): + if fit is None: + return + + fit.oot_baseline_parameter_fit_applied = bool(applied) + fit.oot_baseline_parameter_fit_note = note + fit.oot_baseline_parameter_fit_pre_points = int(pre_points) if pre_points is not None else 0 + fit.oot_baseline_parameter_fit_post_points = int(post_points) if post_points is not None else 0 + fit.oot_baseline_parameter_fit_a0 = a0 + fit.oot_baseline_parameter_fit_a0_error = a0_error + fit.oot_baseline_parameter_fit_a2 = a2 + fit.oot_baseline_parameter_fit_a2_error = a2_error + + +def annotate_pre_detrending_baseline_coefficients( + fit, + source=None, + scale_parameter=None, + scale_value=None, + scale_error=None, + a2_value=None, + a2_error=None, +): + """Retain the measured baseline coefficients that preceded detrending.""" + if fit is None: + return + + fit.pre_detrending_baseline_source = source + fit.pre_detrending_baseline_scale_parameter = scale_parameter + fit.pre_detrending_baseline_scale_value = scale_value + fit.pre_detrending_baseline_scale_error = scale_error + fit.pre_detrending_baseline_a2_value = a2_value + fit.pre_detrending_baseline_a2_error = a2_error + + +def annotate_partial_transit_geometry_prior_assumption(fit, payload): + if fit is None: + return + + payload = payload if isinstance(payload, dict) else {} + fit.partial_transit_geometry_prior_assumption_applied = bool(payload.get('applied', False)) + fit.partial_transit_geometry_prior_assumption_mode = payload.get('mode') + fit.partial_transit_geometry_prior_assumption_note = payload.get('note') + fit.partial_transit_geometry_prior_assumption_fixed_parameters = list( + payload.get('fixed_parameters') or [] + ) + fit.partial_transit_geometry_prior_assumption_sampled_parameters = list( + payload.get('sampled_parameters') or [] + ) -def log_info(string, warn=False, error=False): - if error: - print(f"\033[31m {string}\033[0m") - elif warn: - print(f"\033[33m {string}\033[0m") - else: - print(string) - log.debug(string) - return True -# Initialze plate status log -plateStatus = PlateStatus(log_info) +def annotate_final_fit_prefit_refinement( + fit, + applied, + note=None, + baseline_duration_multiplier=FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT, + duration=None, + original_point_count=None, + refined_point_count=None, + trimmed_pre_points=0, + trimmed_post_points=0, + original_tmid_bounds=None, + refined_tmid_bounds=None, +): + if fit is None: + return + + fit.prefit_refinement_applied = bool(applied) + fit.prefit_refinement_note = note + fit.prefit_refinement_baseline_duration_multiplier = float(baseline_duration_multiplier) + fit.prefit_refinement_duration = duration + fit.prefit_refinement_original_point_count = ( + int(original_point_count) if original_point_count is not None else None + ) + fit.prefit_refinement_point_count = ( + int(refined_point_count) if refined_point_count is not None else None + ) + fit.prefit_refinement_trimmed_pre_points = int(trimmed_pre_points or 0) + fit.prefit_refinement_trimmed_post_points = int(trimmed_post_points or 0) + fit.prefit_refinement_original_tmid_bounds = original_tmid_bounds + fit.prefit_refinement_tmid_bounds = refined_tmid_bounds + + +def annotate_nested_tmid_refinement( + fit, + applied, + note=None, + original_tmid_bounds=None, + refined_tmid_bounds=None, +): + if fit is None: + return + + fit.nested_tmid_refinement_applied = bool(applied) + fit.nested_tmid_refinement_note = note + fit.nested_tmid_refinement_original_tmid_bounds = original_tmid_bounds + fit.nested_tmid_refinement_tmid_bounds = refined_tmid_bounds + + +def annotate_duration_prior(fit, duration_prior): + if fit is None: + return + + summary = duration_prior if isinstance(duration_prior, dict) else {} + fit.duration_prior_applied = bool(summary.get('applied', False)) + fit.duration_prior_note = summary.get('note') + fit.duration_prior_expected_duration = coerce_finite_transit_qc_scalar( + summary.get('expected_duration', np.nan) + ) + fit.duration_prior_sigma_log = coerce_finite_transit_qc_scalar( + summary.get('sigma_log_duration', np.nan) + ) + fit.duration_prior_relative_sigma = coerce_finite_transit_qc_scalar( + summary.get('relative_sigma', np.nan) + ) + fit.duration_prior_source = summary.get('source') -def sigma_clip(ogdata, sigma=3, dt=21, po=2): - nanmask = np.isnan(ogdata) - if po < dt <= len(ogdata[~nanmask]): - mdata = savgol_filter(ogdata[~nanmask], window_length=dt, polyorder=po) - # mdata = median_filter(ogdata[~nanmask], dt) - res = ogdata[~nanmask] - mdata - std = np.nanmedian([np.nanstd(np.random.choice(res, 25)) for i in range(100)]) - # std = np.nanstd(res) # biased from large outliers - sigmask = np.abs(res) > sigma * std - nanmask[~nanmask] = sigmask +def annotate_pre_ultranest_transit_coverage(fit, assessment): + if fit is None: + return - return nanmask + assessment = assessment if isinstance(assessment, dict) else {} + fit.pre_ultranest_transit_coverage = dict(assessment) + fit.pre_ultranest_transit_coverage_valid = bool(assessment.get('valid', False)) + fit.pre_ultranest_transit_coverage_status = assessment.get('success_label') + fit.pre_ultranest_transit_coverage_chance = assessment.get('success_chance') + fit.pre_ultranest_transit_coverage_expected_successful = bool( + assessment.get('expected_successful', False) + ) + fit.pre_ultranest_transit_coverage_note = assessment.get('note') + + +def annotate_lightcurve_filter_diagnostics(fit, diagnostics): + if fit is None: + return + + fit.frame_filter_diagnostics = [dict(diagnostic) for diagnostic in (diagnostics or [])] + + +def prepend_lightcurve_filter_diagnostic(fit, diagnostic): + if fit is None or not diagnostic: + return + + existing = getattr(fit, 'frame_filter_diagnostics', []) + diagnostics = [dict(diagnostic)] + if isinstance(existing, list): + diagnostics.extend(dict(item) for item in existing if isinstance(item, dict)) + fit.frame_filter_diagnostics = diagnostics + + +def _fit_residual_percent(fit): + residuals = np.asarray(getattr(fit, 'residuals', np.array([])), dtype=float).reshape(-1) + if residuals.size == 0: + return residuals + + data = np.asarray(getattr(fit, 'data', np.array([])), dtype=float).reshape(-1) + median_flux = np.nanmedian(data) if data.size else np.nan + if not np.isfinite(median_flux) or median_flux == 0: + return residuals + return residuals / median_flux * 100.0 + + +def final_residual_rejection_keep_mask( + fit, + sigma=FINAL_RESIDUAL_REJECTION_SIGMA, + min_required_points=LIGHTCURVE_MIN_VALID_POINTS, + max_clip_iters=FINAL_RESIDUAL_REJECTION_MAX_CLIP_ITERS, +): + residual_percent = _fit_residual_percent(fit) + point_count = int(residual_percent.size) + summary = { + 'enabled': True, + 'applied': False, + 'sigma': float(sigma), + 'input_point_count': point_count, + 'kept_point_count': point_count, + 'rejected_point_count': 0, + 'median_residual_percent': np.nan, + 'stdev_residual_percent': np.nan, + 'clip_iteration_count': 0, + 'clip_iterations': [], + 'note': None, + } + if point_count == 0: + summary['note'] = "Skipped; the first final fit did not provide residuals." + return np.ones(0, dtype=bool), summary + + valid = np.isfinite(residual_percent) + valid_count = int(np.count_nonzero(valid)) + if valid_count < max(LIGHTCURVE_MIN_VALID_POINTS, 2): + summary['note'] = "Skipped; too few finite residuals were available." + return np.ones(point_count, dtype=bool), summary + + keep_mask = valid.copy() + max_clip_iters = int(max(1, max_clip_iters or 1)) + stopped_on_iteration_cap = False + last_rejecting_center = np.nan + last_rejecting_scatter = np.nan + for clip_iteration in range(1, max_clip_iters + 1): + active = keep_mask & valid + active_count = int(np.count_nonzero(active)) + if active_count < max(LIGHTCURVE_MIN_VALID_POINTS, 2): + summary['note'] = "Skipped; too few finite residuals remained during iterative clipping." + return valid.copy(), summary + + center = float(np.nanmedian(residual_percent[active])) + scatter = float(np.nanstd(residual_percent[active] - center, ddof=1)) + summary['median_residual_percent'] = center + summary['stdev_residual_percent'] = scatter + if not np.isfinite(scatter) or scatter <= 0: + summary['note'] = "Stopped; the final-fit residual scatter was not finite." + break + outlier_mask = active & (np.abs(residual_percent - center) > float(sigma) * scatter) + rejected_this_iteration = int(np.count_nonzero(outlier_mask)) + summary['clip_iterations'].append({ + 'iteration': int(clip_iteration), + 'input_point_count': active_count, + 'median_residual_percent': center, + 'stdev_residual_percent': scatter, + 'rejected_point_count': rejected_this_iteration, + }) + summary['clip_iteration_count'] = int(clip_iteration) + if rejected_this_iteration == 0: + break -def exp_offset(hdr, time_unit, exp): - """Returns exposure offset (in days) of more than 0 if headers reveals - the time was estimated at the start of the exposure rather than the middle - """ - if 'start' in hdr.comments[time_unit]: - return exp / (2.0 * 60.0 * 60.0 * 24.0) - return 0.0 + last_rejecting_center = center + last_rejecting_scatter = scatter + candidate_keep_mask = keep_mask & ~outlier_mask + if int(np.count_nonzero(candidate_keep_mask)) < int(min_required_points): + summary['note'] = ( + f"Skipped; residual rejection would leave {int(np.count_nonzero(candidate_keep_mask))} point(s), " + f"but at least {int(min_required_points)} are required." + ) + summary['clip_iterations'][-1]['skipped_for_minimum_points'] = True + summary['kept_point_count'] = point_count + summary['rejected_point_count'] = 0 + return np.ones(point_count, dtype=bool), summary + keep_mask = candidate_keep_mask + else: + stopped_on_iteration_cap = True -def ut_date(hdr, time_unit, exp): - """Converts the Gregorian Date to Julian Date from the header and returns it - along with the exposure offset - """ - if time_unit == 'DATE-OBS': - greg_date = hdr[time_unit] if 'T' in hdr[time_unit] else f"{hdr[time_unit]}T{hdr['TIME-OBS']}" + kept_count = int(np.count_nonzero(keep_mask)) + rejected_count = int(point_count - kept_count) + summary['kept_point_count'] = kept_count + summary['rejected_point_count'] = rejected_count + + if rejected_count == 0: + summary['note'] = ( + f"No residual outliers exceeded {float(sigma):.1f} sigma from the iterated median residual " + f"({summary['median_residual_percent']:.4f}%, stdev={summary['stdev_residual_percent']:.4f}%)." + ) + return keep_mask, summary + + summary['applied'] = True + if np.isfinite(last_rejecting_center): + summary['median_residual_percent'] = last_rejecting_center + if np.isfinite(last_rejecting_scatter): + summary['stdev_residual_percent'] = last_rejecting_scatter + cap_text = f" after reaching {max_clip_iters} clip iteration(s)" if stopped_on_iteration_cap else "" + summary['note'] = ( + f"Rejected {rejected_count}/{point_count} final-fit residual outlier(s) with iterative " + f"{float(sigma):.1f}-sigma clipping from the median residual " + f"({summary['median_residual_percent']:.4f}%, stdev={summary['stdev_residual_percent']:.4f}%){cap_text}." + ) + return keep_mask, summary + + +def _fit_array_for_rejection_plot(fit, attr_name, fallback=None): + values = getattr(fit, attr_name, fallback) + if values is None: + values = fallback + if values is None: + return np.array([], dtype=float) + return np.asarray(values, dtype=float).reshape(-1) + + +def build_final_residual_rejection_payload(fit, keep_mask, summary, source_indices=None): + keep_mask = np.asarray(keep_mask, dtype=bool).reshape(-1) + rejected_mask = ~keep_mask + time_values = _fit_array_for_rejection_plot(fit, 'time') + phase_values = _fit_array_for_rejection_plot(fit, 'phase') + data_values = _fit_array_for_rejection_plot(fit, 'data') + flux_values = _fit_array_for_rejection_plot(fit, 'detrended', fallback=data_values) + residual_percent = _fit_residual_percent(fit) + + plot_count = min( + keep_mask.size, + time_values.size, + phase_values.size, + flux_values.size, + residual_percent.size, + ) + if plot_count == 0: + rejected_mask = np.zeros(0, dtype=bool) else: - greg_date = hdr[time_unit] + rejected_mask = rejected_mask[:plot_count] + time_values = time_values[:plot_count] + phase_values = phase_values[:plot_count] + flux_values = flux_values[:plot_count] + residual_percent = residual_percent[:plot_count] + + payload = dict(summary or {}) + payload['rejected_time'] = time_values[rejected_mask].tolist() + payload['rejected_phase'] = phase_values[rejected_mask].tolist() + payload['rejected_flux'] = flux_values[rejected_mask].tolist() + payload['rejected_residual_percent'] = residual_percent[rejected_mask].tolist() + if source_indices is not None: + index_values = np.asarray(source_indices, dtype=int).reshape(-1) + if index_values.size >= plot_count: + payload['rejected_source_indices'] = index_values[:plot_count][rejected_mask].tolist() + return payload + + +def initialize_final_residual_rejection_payload( + enabled=True, + input_point_count=0, + sigma=FINAL_RESIDUAL_REJECTION_SIGMA, + note=None, +): + point_count = int(input_point_count or 0) + return { + 'enabled': bool(enabled), + 'applied': False, + 'sigma': float(sigma), + 'input_point_count': point_count, + 'kept_point_count': point_count, + 'rejected_point_count': 0, + 'median_residual_percent': np.nan, + 'stdev_residual_percent': np.nan, + 'clip_iteration_count': 0, + 'clip_iterations': [], + 'refit_iteration_count': 0, + 'refit_iterations': [], + 'rejected_time': [], + 'rejected_phase': [], + 'rejected_flux': [], + 'rejected_residual_percent': [], + 'rejected_source_indices': [], + 'note': note, + } - dt = dup.parse(greg_date) - atime = Time(dt) - julian_time = atime.jd - offset = exp_offset(hdr, time_unit, exp) +def _extend_final_residual_rejection_payload_list(payload, key, values): + existing = payload.get(key) + if not isinstance(existing, list): + existing = [] + if values is None: + values = [] + elif isinstance(values, np.ndarray): + values = values.tolist() + elif not isinstance(values, list): + values = list(values) + payload[key] = existing + values + + +def record_final_residual_rejection_refit_cycle(payload, cycle_payload, refit_iteration): + payload = dict(payload or {}) + cycle_payload = dict(cycle_payload or {}) + rejected_count = int(cycle_payload.get('rejected_point_count', 0) or 0) + payload['enabled'] = True + payload['applied'] = bool(payload.get('applied', False) or rejected_count > 0) + payload['sigma'] = float(cycle_payload.get('sigma', payload.get('sigma', FINAL_RESIDUAL_REJECTION_SIGMA))) + payload['kept_point_count'] = int(cycle_payload.get('kept_point_count', payload.get('kept_point_count', 0)) or 0) + payload['rejected_point_count'] = int(payload.get('rejected_point_count', 0) or 0) + rejected_count + payload['median_residual_percent'] = cycle_payload.get( + 'median_residual_percent', + payload.get('median_residual_percent', np.nan), + ) + payload['stdev_residual_percent'] = cycle_payload.get( + 'stdev_residual_percent', + payload.get('stdev_residual_percent', np.nan), + ) + payload['clip_iteration_count'] = int(payload.get('clip_iteration_count', 0) or 0) + int( + cycle_payload.get('clip_iteration_count', 0) or 0 + ) + _extend_final_residual_rejection_payload_list( + payload, + 'clip_iterations', + cycle_payload.get('clip_iterations', []), + ) + for key in ( + 'rejected_time', + 'rejected_phase', + 'rejected_flux', + 'rejected_residual_percent', + 'rejected_source_indices', + ): + _extend_final_residual_rejection_payload_list(payload, key, cycle_payload.get(key, [])) + + refit_iterations = payload.get('refit_iterations') + if not isinstance(refit_iterations, list): + refit_iterations = [] + refit_iterations.append({ + 'iteration': int(refit_iteration), + 'input_point_count': int(cycle_payload.get('input_point_count', 0) or 0), + 'kept_point_count': int(cycle_payload.get('kept_point_count', 0) or 0), + 'rejected_point_count': rejected_count, + 'median_residual_percent': cycle_payload.get('median_residual_percent', np.nan), + 'stdev_residual_percent': cycle_payload.get('stdev_residual_percent', np.nan), + 'clip_iteration_count': int(cycle_payload.get('clip_iteration_count', 0) or 0), + 'clip_iterations': list(cycle_payload.get('clip_iterations', [])), + 'note': cycle_payload.get('note'), + }) + payload['refit_iterations'] = refit_iterations + payload['refit_iteration_count'] = int(len(refit_iterations)) + return payload + + +def update_final_residual_rejection_final_pass(payload, final_summary): + payload = dict(payload or {}) + final_summary = dict(final_summary or {}) + payload['final_clip_summary'] = final_summary + payload['median_residual_percent'] = final_summary.get( + 'median_residual_percent', + payload.get('median_residual_percent', np.nan), + ) + payload['stdev_residual_percent'] = final_summary.get( + 'stdev_residual_percent', + payload.get('stdev_residual_percent', np.nan), + ) + return payload + + +def finalize_final_residual_rejection_payload( + payload, + current_point_count=None, + stopped_reason=None, +): + payload = dict(payload or {}) + if current_point_count is not None: + payload['kept_point_count'] = int(current_point_count) + + rejected_count = int(payload.get('rejected_point_count', 0) or 0) + input_point_count = int(payload.get('input_point_count', payload.get('kept_point_count', 0)) or 0) + refit_count = int(payload.get('refit_iteration_count', 0) or 0) + payload['applied'] = bool(payload.get('enabled', True) and rejected_count > 0) + if not payload.get('enabled', True): + if not payload.get('note'): + payload['note'] = "Disabled per optional_info setting." + return payload + + if rejected_count <= 0: + if payload.get('note') is None: + payload['note'] = "No final-fit residual outliers were rejected." + return payload + + stop_text = " Final pass found no new residual outliers." + if stopped_reason == 'max_refits': + stop_text = f" Stopped after reaching {FINAL_RESIDUAL_REJECTION_MAX_REFITS} refit cycle(s)." + elif stopped_reason == 'refit_failed': + stop_text = " Stopped because the next residual-rejected UltraNest refit did not converge." + elif stopped_reason == 'shape_mismatch': + stop_text = " Stopped because the next residual array did not align with the light-curve points." + + payload['note'] = ( + f"Rejected {rejected_count}/{input_point_count} final-fit residual outlier(s) over " + f"{refit_count} iterative UltraNest refit cycle(s) using " + f"{float(payload.get('sigma', FINAL_RESIDUAL_REJECTION_SIGMA)):.1f}-sigma median clipping." + f"{stop_text}" + ) + return payload + + +def annotate_final_residual_rejection(fit, payload): + if fit is None: + return + + payload = dict(payload or {}) + fit.final_residual_rejection = payload + fit.final_residual_rejection_applied = bool(payload.get('applied', False)) + fit.final_residual_rejection_sigma = payload.get('sigma') + fit.final_residual_rejection_point_count = int(payload.get('input_point_count', 0) or 0) + fit.final_residual_rejection_rejected_count = int(payload.get('rejected_point_count', 0) or 0) + fit.final_residual_rejection_note = payload.get('note') + + +def annotate_selected_photometry_debug( + fit, + times, + target_flux, + comp_flux, + raw_ratio, + initial_sigma_keep_mask, + target_flux_error=None, + comp_flux_error=None, + relative_flux_error=None, + prefit_raw_ratio_keep_mask=None, + phase_clip_keep_mask_on_sigma_filtered=None, +): + if fit is None: + return + + sigma_keep_mask = np.asarray(initial_sigma_keep_mask, dtype=bool) + if prefit_raw_ratio_keep_mask is None: + raw_ratio_keep_mask = np.ones(sigma_keep_mask.shape, dtype=bool) + else: + raw_ratio_keep_mask = np.asarray(prefit_raw_ratio_keep_mask, dtype=bool) + if raw_ratio_keep_mask.shape != sigma_keep_mask.shape: + raw_ratio_keep_mask = np.ones(sigma_keep_mask.shape, dtype=bool) + + prefit_keep_mask = sigma_keep_mask & raw_ratio_keep_mask + prefit_kept_count = int(np.count_nonzero(prefit_keep_mask)) + if phase_clip_keep_mask_on_sigma_filtered is None: + phase_keep_mask = np.ones(prefit_kept_count, dtype=bool) + else: + phase_keep_mask = np.asarray(phase_clip_keep_mask_on_sigma_filtered, dtype=bool) + if phase_keep_mask.shape[0] != prefit_kept_count: + phase_keep_mask = np.ones(prefit_kept_count, dtype=bool) + + fit.selected_photometry_debug = { + 'times': np.asarray(times, dtype=float).copy(), + 'target_flux': np.asarray(target_flux, dtype=float).copy(), + 'comp_flux': np.asarray(comp_flux, dtype=float).copy(), + 'raw_ratio': np.asarray(raw_ratio, dtype=float).copy(), + 'target_flux_error': np.asarray(target_flux_error, dtype=float).copy() + if target_flux_error is not None else np.full(sigma_keep_mask.shape, np.nan, dtype=float), + 'comp_flux_error': np.asarray(comp_flux_error, dtype=float).copy() + if comp_flux_error is not None else np.full(sigma_keep_mask.shape, np.nan, dtype=float), + 'relative_flux_error': np.asarray(relative_flux_error, dtype=float).copy() + if relative_flux_error is not None else np.full(sigma_keep_mask.shape, np.nan, dtype=float), + 'initial_sigma_keep_mask': sigma_keep_mask.copy(), + 'prefit_raw_ratio_keep_mask': raw_ratio_keep_mask.copy(), + 'phase_clip_keep_mask_on_sigma_filtered': phase_keep_mask.copy(), + } - return julian_time + offset +def transit_qc_airmass_reference(airmass): + values = np.asarray(airmass, dtype=float) + finite = values[np.isfinite(values)] + if finite.size == 0: + return 0.0 + return float(np.nanmean(finite)) -def julian_date(hdr, time_unit, exp): - """Returns Julian Date from the header along with the exposure offset. - If the image is taken from MicroObservatory (MJD-OBS), - add a timing offset (2400000.5) due to being less precise - """ - time_offset = 2400000.5 if time_unit == 'MJD-OBS' else 0.0 - julian_time = float(hdr[time_unit]) + time_offset - offset = exp_offset(hdr, time_unit, exp) +def transit_qc_airmass_trend(a2, airmass, reference=None): + values = np.asarray(airmass, dtype=float) + if reference is None: + reference = transit_qc_airmass_reference(values) + return np.exp(float(a2) * (values - float(reference))) - return julian_time + offset -def get_exp_time(hdr): - exp_list = ["EXPTIME", "EXPOSURE", "EXP"] - exp_time = next((exptime for exptime in exp_list if exptime in hdr), None) - return hdr[exp_time] if exp_time is not None else 0.0 +def solve_transit_qc_flux_baseline(systematics, data, dataerr=None): + systematics = np.asarray(systematics, dtype=float) + data = np.asarray(data, dtype=float) + if systematics.shape != data.shape: + return np.nan -def img_time_jd(hdr): - """Converts time from the header file to the Julian Date (JD, if needed) - and adds an exposure offset (if needed) + weights = np.ones(systematics.shape, dtype=float) + if dataerr is not None: + dataerr = np.asarray(dataerr, dtype=float) + if dataerr.shape != data.shape: + dataerr = None + else: + weights = np.zeros(systematics.shape, dtype=float) + valid_err = np.isfinite(dataerr) & (dataerr > 0) + weights[valid_err] = 1.0 / (dataerr[valid_err] ** 2) + + mask = np.isfinite(systematics) & np.isfinite(data) & (systematics != 0) + if dataerr is not None: + mask &= np.isfinite(weights) & (weights > 0) + + if not np.any(mask): + return np.nan + + masked_systematics = systematics[mask] + masked_data = data[mask] + masked_weights = weights[mask] + denom = np.sum(masked_weights * masked_systematics ** 2) + if np.isfinite(denom) and denom > 0: + baseline = np.sum(masked_weights * masked_data * masked_systematics) / denom + if np.isfinite(baseline): + return float(baseline) + + ratio = masked_data / masked_systematics + ratio = ratio[np.isfinite(ratio)] + if ratio.size == 0: + return np.nan + return float(np.nanmedian(ratio)) + + +def compute_transit_qc_model_chi2(data, model, dataerr=None): + data = np.asarray(data, dtype=float) + model = np.asarray(model, dtype=float) + if data.shape != model.shape: + return np.nan, 0 + + mask = np.isfinite(data) & np.isfinite(model) + if dataerr is not None: + dataerr = np.asarray(dataerr, dtype=float) + if dataerr.shape != data.shape: + dataerr = None + else: + mask &= np.isfinite(dataerr) & (dataerr > 0) - Parameters - ---------- - hdr : astropy.io.fits.header.Header - A header file that includes the time from when the image was taken - Returns - ------- - float - Time of when the image was taken in the JD with exposure offset - """ - time_list = ['UT-OBS', 'JULIAN', 'MJD-OBS', 'DATE-OBS'] + point_count = int(np.count_nonzero(mask)) + if point_count == 0: + return np.nan, 0 - exp = get_exp_time(hdr) - hdr_time = next((time_unit for time_unit in time_list if time_unit in hdr), None) + residuals = data[mask] - model[mask] + if dataerr is not None: + chi2 = np.sum((residuals / dataerr[mask]) ** 2) + else: + chi2 = np.sum(residuals ** 2) + return float(chi2), point_count - if hdr_time == 'MJD_OBS': - hdr_time = hdr_time if "epoch" not in hdr.comments[hdr_time] else 'DATE-OBS' - if hdr_time in ['UT-OBS', 'DATE-OBS']: - return ut_date(hdr, hdr_time, exp) - return julian_date(hdr, hdr_time, exp) +def compute_transit_qc_bic(chi2, point_count, parameter_count): + try: + chi2 = float(chi2) + point_count = int(point_count) + parameter_count = int(parameter_count) + except (TypeError, ValueError): + return np.nan + if not np.isfinite(chi2) or point_count <= 0 or parameter_count <= 0: + return np.nan + return float(chi2 + parameter_count * np.log(point_count)) -def img_time_bjd_tdb(hdr, p_dict, info_dict): - """Converts time from the header file to BJD-TDB time (if needed) - and adds an exposure offset (if needed) - Parameters - ---------- - hdr : astropy.io.fits.header.Header - A header file that includes the time from when the image was taken - p_dict: planetary settings dictionary - info_dict: observatory settings dictionary +def clip_unit_interval(value): + try: + numeric_value = float(value) + except (TypeError, ValueError): + return np.nan - Returns - ------- - float - Time of when the image was taken in BJD-TDB with exposure offset - """ - # Check for BJD time first (preference) - time_list = ['BJD_TDB', 'BJD_TBD', 'BJD'] - exp = get_exp_time(hdr) + if not np.isfinite(numeric_value): + return np.nan + return float(np.clip(numeric_value, 0.0, 1.0)) - hdr_time = next((time for time in time_list if time in hdr), None) - # Not found, get julian date - if hdr_time is None: - time_list = ['UT-OBS', 'JULIAN', 'MJD-OBS', 'DATE-OBS'] - hdr_time = next((time for time in time_list if time in hdr), None) - if hdr_time == 'MJD_OBS': - hdr_time = hdr_time if "epoch" not in hdr.comments[hdr_time] else 'DATE-OBS' - if hdr_time in ['UT-OBS', 'DATE-OBS']: - jd_time = ut_date(hdr, hdr_time, exp) - else: - jd_time = julian_date(hdr, hdr_time, exp) - # And convert to BJD_TDB - bjd_time = convert_jd_to_bjd([jd_time], p_dict, info_dict)[0] - else: # Else, already BJD - convert and adjust for exposure - bjd_time = julian_date(hdr, hdr_time, exp) - return bjd_time +def transit_qc_residual_scatter(data, model): + data = np.asarray(data, dtype=float) + model = np.asarray(model, dtype=float) + if data.shape != model.shape or data.size == 0: + return np.nan -def air_mass(hdr, ra, dec, lat, long, elevation, time): - """Scrapes or calculates the airmass at the time of when the image was taken. - Airmass(X): X = sec(z), z = secant of the zenith angle (angle between zenith and star) + mask = np.isfinite(data) & np.isfinite(model) + if not np.any(mask): + return np.nan - Parameters - ---------- - hdr : astropy.io.fits.header.Header - A header file that may include the airmass or altitude from when the image was taken - ra : float - Right Ascension - dec : float - Declination - lat : float - Latitude - long : float - Longitude - elevation : float - Elevation/Altitude + median_flux = np.nanmedian(data[mask]) + if not np.isfinite(median_flux) or median_flux == 0: + return np.nan - Returns - ------- - float - Airmass value - """ - if 'AIRMASS' in hdr: - am = float(hdr['AIRMASS']) - elif 'TELALT' in hdr: - alt = float(hdr['TELALT']) - cos_am = np.cos((np.pi / 180) * (90.0 - alt)) - am = 1 / cos_am - else: - pointing = SkyCoord(f"{ra} {dec}", unit=(u.deg, u.deg), frame='icrs') + residuals = data[mask] - model[mask] + return float(np.std(residuals) / median_flux) - location = EarthLocation.from_geodetic(lat=lat * u.deg, lon=long * u.deg, height=elevation) - time = Time(time, format='jd', scale='utc', location=location) - point_altaz = pointing.transform_to(AltAz(obstime=time, location=location)) - am = float(point_altaz.secz) - return am +def transit_qc_model_depth_fraction(model): + model = np.asarray(model, dtype=float) + if model.ndim != 1 or model.size == 0: + return np.nan -def flux_conversion(fluxes, errors, flux_format): - """Converting differential magnitudes to fluxes and calculating its errors - """ - conv = 1000.0 if flux_format == 'millimagnitude' else 1.0 + finite = model[np.isfinite(model)] + if finite.size == 0: + return np.nan - pos_err = 10.0 ** (-0.4 * ((fluxes + errors) / conv)) - neg_err = 10.0 ** (-0.4 * ((fluxes - errors) / conv)) - fluxes = 10.0 ** (-0.4 * (fluxes / conv)) + baseline = float(np.nanpercentile(finite, 95.0)) + minimum = float(np.nanmin(finite)) + if not np.isfinite(baseline) or not np.isfinite(minimum) or baseline <= 0: + return np.nan - pos_err_dist = abs(pos_err - fluxes) - neg_err_dist = abs(neg_err - fluxes) - mean_errors = (pos_err_dist * neg_err_dist) ** 0.5 + depth = (baseline - minimum) / baseline + return float(depth) if np.isfinite(depth) and depth > 0 else np.nan - return fluxes, mean_errors +def transit_qc_deviation_score_from_sigma(sigma_offset, sigma_threshold): + try: + sigma_offset = abs(float(sigma_offset)) + sigma_threshold = float(sigma_threshold) + except (TypeError, ValueError): + return np.nan -# Check for difference between NEA and initialization file -def check_parameters(init_parameters, parameters): - different = False - uncert = 1 / 36 + if not np.isfinite(sigma_offset) or not np.isfinite(sigma_threshold) or sigma_threshold <= 0: + return np.nan - for key, value in parameters.items(): - if key in ['ra', 'dec'] and init_parameters[key]: - if not parameters[key] - uncert <= init_parameters[key] <= parameters[key] + uncert: - different = True - break - continue - if value != init_parameters[key]: - different = True - break + return float(max(0.0, 1.0 - sigma_offset / sigma_threshold)) - if different: - log_info("\nDifference(s) found between initialization file parameters and " - "those scraped by EXOTIC from the NASA Exoplanet Archive." - "\nWould you like:" - "\n (1) EXOTIC to adopt of all of your defined parameters or" - "\n (2) to review the ones scraped from the Archive that differ?") - opt = user_input("Enter 1 or 2: ", type_=int, values=[1, 2]) - if opt == 2: - return True - else: - return False +def transit_qc_rprs_deviation_uncertainty(fitted_rprs_unc, expected_rprs_unc, expected_rprs): + terms = [] + for value in (fitted_rprs_unc, expected_rprs_unc): + value = coerce_finite_transit_qc_scalar(value) + if np.isfinite(value) and value > 0: + terms.append(float(value)) + systematic_floor = np.nan + expected_rprs = coerce_finite_transit_qc_scalar(expected_rprs) + if np.isfinite(expected_rprs) and expected_rprs > 0: + systematic_floor = float( + TRANSIT_QC_RPRS_DEVIATION_SYSTEMATIC_FLOOR_FRACTION * abs(expected_rprs) + ) + if systematic_floor > 0: + terms.append(systematic_floor) -# --------PLANETARY PARAMETERS UI------------------------------------------ -# Get the user's confirmation of values that will later be used in lightcurve fit -def get_planetary_parameters(candplanetbool, userpdict, pdict=None): - log_info("*******************************************") - log_info("Planetary Parameters for Lightcurve Fitting") + if not terms: + return np.nan, systematic_floor - # The order of planet_params list must match the pDict that is declared when scraping the NASA Exoplanet Archive - planet_params = ["Target Star RA in the form: HH:MM:SS (ignore the decimal values)", - "Target Star DEC in form: DD:MM:SS (ignore the decimal values and don't forget the '+' or '-' sign!)", - "Planet's Name", - "Host Star's Name", - "Orbital Period (days)", - "Orbital Period Uncertainty (days) \n(Keep in mind that 1.2e-34 is the same as 1.2 x 10^-34)", - "Published Mid-Transit Time (BJD_UTC)", - "Mid-Transit Time Uncertainty (BJD-UTC)", - "Ratio of Planet to Stellar Radius (Rp/Rs)", - "Ratio of Planet to Stellar Radius (Rp/Rs) Uncertainty", - "Ratio of Distance to Stellar Radius (a/Rs)", - "Ratio of Distance to Stellar Radius (a/Rs) Uncertainty", - "Orbital Inclination (deg)", - "Orbital Inclination (deg) Uncertainty", - "Argument of Periastron (deg)", - "Orbital Eccentricity (0 if null)", - "Star Effective Temperature (K)", - "Star Effective Temperature Positive Uncertainty (K)", - "Star Effective Temperature Negative Uncertainty (K)", - "Star Metallicity ([FE/H])", - "Star Metallicity Positive Uncertainty ([FE/H])", - "Star Metallicity Negative Uncertainty ([FE/H])", - "Star Surface Gravity (log(g))", - "Star Surface Gravity Positive Uncertainty (log(g))", - "Star Surface Gravity Negative Uncertainty (log(g))", - "Star Distance (pc)", - "Star Proper Motion RA (mas/yr)", - "Star Proper Motion DEC (mas/yr)"] + return float(np.sqrt(np.sum(np.square(terms)))), systematic_floor - # Conversion between hours to degrees if user entered ra and dec - if userpdict['ra'] is None: - userpdict['ra'] = user_input(f"\nEnter the {planet_params[0]}: ", type_=str) - if userpdict['dec'] is None: - userpdict['dec'] = user_input(f"\nEnter the {planet_params[1]}: ", type_=str) - if type(userpdict['ra']) and type(userpdict['dec']) is str: - userpdict['ra'], userpdict['dec'] = radec_hours_to_degree(userpdict['ra'], userpdict['dec']) - radeclist = ['ra', 'dec'] - if not candplanetbool: - for idx, item in enumerate(radeclist): - uncert = 20 / 3600 - if pdict[item] - uncert <= userpdict[item] <= pdict[item] + uncert: - continue - else: - log_info(f"\n\nWarning: {pdict['pName']} initialization file's {planet_params[idx]} does not match " - "the value scraped by EXOTIC from the NASA Exoplanet Archive.\n", warn=True) - log_info(f"\tNASA Exoplanet Archive value (degrees): {pdict[item]}", warn=True) - log_info(f"\tInitialization file value (degrees): {userpdict[item]}", warn=True) - log_info("\nWould you like to:" - "\n (1) use NASA Exoplanet Archive value, " - "\n (2) use initialization file value, or " - "\n (3) enter in a new value.", warn=True) - option = user_input("Which option do you choose? (1/2/3): ", type_=int, values=[1, 2, 3]) +def transit_qc_duration_score(duration_ratio): + try: + duration_ratio = float(duration_ratio) + except (TypeError, ValueError): + return np.nan - if option == 1: - userpdict[item] = pdict[item] - elif option == 2: - continue - else: - userpdict['ra'] = user_input(f"Enter the {planet_params[0]}: ", type_=str) - userpdict['dec'] = user_input(f"Enter the {planet_params[1]}: ", type_=str) - break + if not np.isfinite(duration_ratio) or duration_ratio <= 0: + return np.nan - if type(userpdict['ra']) and type(userpdict['dec']) is str: - userpdict['ra'], userpdict['dec'] = radec_hours_to_degree(userpdict['ra'], userpdict['dec']) + max_log_deviation = np.log(TRANSIT_QC_DURATION_RATIO_MAX) + if not np.isfinite(max_log_deviation) or max_log_deviation <= 0: + return np.nan - # Exoplanet confirmed in NASA Exoplanet Archive - if not candplanetbool: + score = 1.0 - abs(np.log(duration_ratio)) / max_log_deviation + return float(np.clip(score, 0.0, 1.0)) + + +def transit_qc_geometry_contact_duration(parameters, contact_radius): + parameters = parameters or {} + try: + period = float(parameters.get('per', np.nan)) + ars = float(parameters.get('ars', np.nan)) + inc = float(parameters.get('inc', np.nan)) + contact_radius = float(contact_radius) + except (TypeError, ValueError): + return np.nan + + if ( + not np.isfinite(period) or period <= 0 + or not np.isfinite(ars) or ars <= 0 + or not np.isfinite(inc) + or not np.isfinite(contact_radius) or contact_radius <= 0 + ): + return np.nan + + ecc = coerce_finite_transit_qc_scalar(parameters.get('ecc', 0.0)) + omega = np.deg2rad(coerce_finite_transit_qc_scalar(parameters.get('omega', 0.0))) + denominator = 1.0 + ecc * np.sin(omega) + if not np.isfinite(denominator) or np.isclose(denominator, 0.0): + return np.nan + + impact_scale = ars * (1.0 - ecc ** 2) / denominator + inc_rad = np.deg2rad(inc) + sin_inc = np.sin(inc_rad) + if not np.isfinite(impact_scale) or impact_scale <= 0 or not np.isfinite(sin_inc) or sin_inc <= 0: + return np.nan + + impact_parameter = impact_scale * np.cos(inc_rad) + chord_sq = contact_radius ** 2 - impact_parameter ** 2 + if not np.isfinite(chord_sq) or chord_sq <= 0: + return np.nan + + argument = np.sqrt(chord_sq) / (ars * sin_inc) + if not np.isfinite(argument): + return np.nan + argument = float(np.clip(argument, -1.0, 1.0)) + eccentric_speed_factor = np.sqrt(1.0 - ecc ** 2) / denominator + duration = (period / np.pi) * np.arcsin(argument) * eccentric_speed_factor + return float(duration) if np.isfinite(duration) and duration > 0 else np.nan + + +def transit_qc_sampling_summary(fit): + summary = { + 'available': False, + 'score': np.nan, + 'ingress_count': 0, + 'egress_count': 0, + 'in_transit_count': 0, + 'pre_baseline_count': 0, + 'post_baseline_count': 0, + 'total_duration': np.nan, + 'ingress_duration': np.nan, + 'detail': 'sampling unavailable', + } + if fit is None: + return summary + + times = np.asarray(getattr(fit, 'time', []), dtype=float) + parameters = getattr(fit, 'parameters', {}) or {} + if times.ndim != 1 or times.size == 0: + return summary + + tmid = coerce_finite_transit_qc_scalar(parameters.get('tmid', np.nan)) + rprs = coerce_finite_transit_qc_scalar(parameters.get('rprs', np.nan)) + if not np.isfinite(tmid) or not np.isfinite(rprs) or rprs < 0: + return summary + + total_duration = transit_qc_geometry_contact_duration(parameters, 1.0 + rprs) + full_duration = transit_qc_geometry_contact_duration(parameters, max(1.0 - rprs, 0.0)) + if not np.isfinite(total_duration) or total_duration <= 0: + total_duration = coerce_finite_transit_qc_scalar(getattr(fit, 'duration_expected', np.nan)) + if not np.isfinite(total_duration) or total_duration <= 0: + return summary + + if np.isfinite(full_duration) and full_duration >= 0 and full_duration < total_duration: + ingress_duration = 0.5 * (total_duration - full_duration) + else: + ingress_duration = 0.2 * total_duration + if not np.isfinite(ingress_duration) or ingress_duration <= 0: + return summary + ingress_duration = min(float(ingress_duration), 0.5 * float(total_duration)) + + finite_times = times[np.isfinite(times)] + if finite_times.size == 0: + return summary + + start = tmid - 0.5 * total_duration + end = tmid + 0.5 * total_duration + ingress_end = min(start + ingress_duration, tmid) + egress_start = max(end - ingress_duration, tmid) + + ingress_count = int(np.count_nonzero((finite_times >= start) & (finite_times <= ingress_end))) + egress_count = int(np.count_nonzero((finite_times >= egress_start) & (finite_times <= end))) + in_transit_count = int(np.count_nonzero((finite_times >= start) & (finite_times <= end))) + pre_baseline_count = int(np.count_nonzero(finite_times < start)) + post_baseline_count = int(np.count_nonzero(finite_times > end)) + + ingress_egress_score = min(min(ingress_count, egress_count) / 4.0, 1.0) + in_transit_score = min(in_transit_count / 20.0, 1.0) + baseline_score = min(min(pre_baseline_count, post_baseline_count) / 12.0, 1.0) + score = float(np.clip( + 0.60 * ingress_egress_score + + 0.25 * in_transit_score + + 0.15 * baseline_score, + 0.0, + 1.0, + )) + + summary.update({ + 'available': True, + 'score': score, + 'ingress_count': ingress_count, + 'egress_count': egress_count, + 'in_transit_count': in_transit_count, + 'pre_baseline_count': pre_baseline_count, + 'post_baseline_count': post_baseline_count, + 'total_duration': float(total_duration), + 'ingress_duration': float(ingress_duration), + 'detail': ( + f"ingress={ingress_count}, egress={egress_count}, " + f"in-transit={in_transit_count}, baseline pre/post={pre_baseline_count}/{post_baseline_count}" + ), + }) + return summary + + +def estimate_midpoint_anchored_partial_duration(fit, assessment): + times = np.asarray(getattr(fit, 'time', []), dtype=float) + transit_model = np.asarray(getattr(fit, 'transit', []), dtype=float) + if times.shape != transit_model.shape or times.size < 2: + return np.nan + + parameters = getattr(fit, 'parameters', {}) or {} + tmid = coerce_finite_transit_qc_scalar(parameters.get('tmid', assessment.get('expected_tmid', np.nan))) + if not np.isfinite(tmid): + return np.nan + + finite_times = np.sort(times[np.isfinite(times)]) + if finite_times.size < 2: + return np.nan + cadence = float(np.nanmedian(np.diff(finite_times))) + if not np.isfinite(cadence) or cadence <= 0: + return np.nan + + in_transit = np.isfinite(times) & np.isfinite(transit_model) & (transit_model < 1.0) + if not np.any(in_transit): + return np.nan + + covers_ingress = bool(assessment.get('covers_ingress', False)) + covers_egress = bool(assessment.get('covers_egress', False)) + if covers_ingress and not covers_egress: + side_times = times[in_transit & (times <= tmid)] + if side_times.size == 0: + return np.nan + half_duration = tmid - float(np.nanmin(side_times)) + 0.5 * cadence + elif covers_egress and not covers_ingress: + side_times = times[in_transit & (times >= tmid)] + if side_times.size == 0: + return np.nan + half_duration = float(np.nanmax(side_times)) - tmid + 0.5 * cadence + else: + return np.nan + + if not np.isfinite(half_duration) or half_duration <= 0: + return np.nan + return float(2.0 * half_duration) + + +def transit_qc_duration_consistency_measurement(fit): + duration_measured = getattr(fit, 'duration_measured', np.nan) + assessment = getattr(fit, 'pre_ultranest_transit_coverage', None) + if not isinstance(assessment, dict) or not assessment.get('valid', False): + return duration_measured, True, None + + covers_ingress = bool(assessment.get('covers_ingress', False)) + covers_mid_transit = bool(assessment.get('covers_mid_transit', False)) + covers_egress = bool(assessment.get('covers_egress', False)) + if covers_ingress and covers_egress: + return duration_measured, True, None + + observed_segment = assessment.get('observed_segment') or 'partial transit' + if covers_mid_transit and (covers_ingress or covers_egress): + partial_duration = estimate_midpoint_anchored_partial_duration(fit, assessment) + if np.isfinite(partial_duration) and partial_duration > 0: + edge = 'ingress' if covers_ingress else 'egress' + note = ( + "Duration consistency used a midpoint-anchored partial estimate: fitted Tmid to " + f"observed {edge}, doubled to estimate the full duration ({observed_segment})." + ) + return partial_duration, True, note + + note = ( + "Duration consistency was not scored because the expected transit was only partially " + f"observed ({observed_segment}) and the midpoint-to-edge duration could not be measured." + ) + return np.nan, False, note + + note = ( + "Duration consistency was not scored because the expected transit was only partially " + f"observed ({observed_segment}); either both ingress and egress, or mid-transit plus " + "one transit edge, are needed to estimate duration." + ) + return np.nan, False, note + + +def transit_qc_saturating_score(value, scale): + try: + value = float(value) + scale = float(scale) + except (TypeError, ValueError): + return np.nan + + if not np.isfinite(value) or not np.isfinite(scale) or scale <= 0: + return np.nan + + return float(np.clip(1.0 - np.exp(-max(value, 0.0) / scale), 0.0, 1.0)) + + +def transit_qc_residual_scatter_score( + residual_scatter, + transit_depth=np.nan, + full_credit_ratio=0.5, + zero_credit_ratio=4.0, + decay_rate=3.0, +): + try: + residual_scatter = float(residual_scatter) + transit_depth = float(transit_depth) + full_credit_ratio = float(full_credit_ratio) + zero_credit_ratio = float(zero_credit_ratio) + decay_rate = float(decay_rate) + except (TypeError, ValueError): + return np.nan + + if ( + not np.isfinite(residual_scatter) + or residual_scatter < 0 + or not np.isfinite(full_credit_ratio) + or full_credit_ratio < 0 + or not np.isfinite(zero_credit_ratio) + or zero_credit_ratio <= full_credit_ratio + or not np.isfinite(decay_rate) + or decay_rate <= 0 + ): + return np.nan + + if not np.isfinite(transit_depth) or transit_depth <= 0: + return np.nan + + scatter_ratio = residual_scatter / transit_depth + if scatter_ratio <= full_credit_ratio: + return 1.0 + if scatter_ratio >= zero_credit_ratio: + return 0.0 + + interval_fraction = ( + (scatter_ratio - full_credit_ratio) + / (zero_credit_ratio - full_credit_ratio) + ) + numerator = np.exp(-decay_rate * interval_fraction) - np.exp(-decay_rate) + denominator = 1.0 - np.exp(-decay_rate) + return float(np.clip(numerator / denominator, 0.0, 1.0)) + + +def transit_qc_flatness_declining_score(value, full_credit, zero_credit, decay_rate=3.0): + try: + value = float(value) + full_credit = float(full_credit) + zero_credit = float(zero_credit) + decay_rate = float(decay_rate) + except (TypeError, ValueError): + return np.nan + + if ( + not np.isfinite(value) + or value < 0 + or not np.isfinite(full_credit) + or full_credit < 0 + or not np.isfinite(zero_credit) + or zero_credit <= full_credit + or not np.isfinite(decay_rate) + or decay_rate <= 0 + ): + return np.nan + + if value <= full_credit: + return 1.0 + if value >= zero_credit: + return 0.0 + + interval_fraction = (value - full_credit) / (zero_credit - full_credit) + numerator = np.exp(-decay_rate * interval_fraction) - np.exp(-decay_rate) + denominator = 1.0 - np.exp(-decay_rate) + return float(np.clip(numerator / denominator, 0.0, 1.0)) + + +def robust_sigma(values): + values = np.asarray(values, dtype=float) + finite = values[np.isfinite(values)] + if finite.size == 0: + return np.nan + + center = float(np.nanmedian(finite)) + mad = float(np.nanmedian(np.abs(finite - center))) + if np.isfinite(mad) and mad > 0: + return float(1.4826 * mad) + + scatter = float(np.nanstd(finite, ddof=1)) if finite.size > 1 else 0.0 + return scatter if np.isfinite(scatter) else np.nan + + +def transit_qc_residual_flatness_summary(residuals, coordinates=None, min_points=12): + summary = { + 'available': False, + 'score': np.nan, + 'point_count': 0, + 'scatter': np.nan, + 'trend_strength': np.nan, + 'curve_strength': np.nan, + 'scatter_ratio': np.nan, + 'zero_offset_strength': np.nan, + 'sign_imbalance': np.nan, + 'zero_bias_score': np.nan, + 'trend_score': np.nan, + 'curve_score': np.nan, + 'scatter_stability_score': np.nan, + 'dominant': None, + 'detail': 'residual flatness unavailable', + } + + residuals = np.asarray(residuals, dtype=float).reshape(-1) + if residuals.size == 0: + return summary + + if coordinates is None: + coordinates = np.arange(residuals.size, dtype=float) + else: + coordinates = np.asarray(coordinates, dtype=float).reshape(-1) + if coordinates.shape != residuals.shape: + coordinates = np.arange(residuals.size, dtype=float) + + finite = np.isfinite(residuals) & np.isfinite(coordinates) + point_count = int(np.count_nonzero(finite)) + summary['point_count'] = point_count + if point_count < int(min_points): + summary['detail'] = f"not enough residual points ({point_count} < {int(min_points)})" + return summary + + residuals = residuals[finite] + coordinates = coordinates[finite] + order = np.argsort(coordinates) + residuals = residuals[order] + coordinates = coordinates[order] + + median_residual = float(np.nanmedian(residuals)) + centered = residuals - median_residual + scatter = robust_sigma(centered) + if not np.isfinite(scatter): + summary['detail'] = "residual scatter unavailable" + return summary + if scatter <= np.finfo(float).eps: + zero_bias_score = 1.0 if abs(median_residual) <= np.finfo(float).eps else 0.0 + summary.update({ + 'available': True, + 'score': zero_bias_score, + 'scatter': float(scatter), + 'trend_strength': 0.0, + 'curve_strength': 0.0, + 'scatter_ratio': 1.0, + 'zero_offset_strength': 0.0 if zero_bias_score == 1.0 else np.inf, + 'sign_imbalance': 0.0 if zero_bias_score == 1.0 else 1.0, + 'zero_bias_score': zero_bias_score, + 'trend_score': 1.0, + 'curve_score': 1.0, + 'scatter_stability_score': 1.0, + 'dominant': 'flat' if zero_bias_score == 1.0 else 'zero bias', + 'detail': ( + f"flat residuals; n={point_count}" + if zero_bias_score == 1.0 + else f"zero bias limited, median offset=inf, sign imbalance=1.00, n={point_count}" + ), + }) + return summary + + zero_offset_strength = abs(median_residual) / scatter + sign_tolerance = 0.05 * scatter + signed = residuals[np.abs(residuals) > sign_tolerance] + sign_imbalance = np.nan + if signed.size >= max(6, point_count // 3): + positive_fraction = float(np.count_nonzero(signed > 0.0) / signed.size) + sign_imbalance = float(abs(2.0 * positive_fraction - 1.0)) + + normalized = centered / scatter + coordinate_min = float(np.nanmin(coordinates)) + coordinate_max = float(np.nanmax(coordinates)) + if not np.isfinite(coordinate_min) or not np.isfinite(coordinate_max) or coordinate_max <= coordinate_min: + x01 = np.linspace(0.0, 1.0, point_count) + else: + x01 = (coordinates - coordinate_min) / (coordinate_max - coordinate_min) + x = 2.0 * x01 - 1.0 + + trend_strength = np.nan + try: + trend_design = np.column_stack([np.ones(point_count, dtype=float), x]) + trend_coeff, *_ = np.linalg.lstsq(trend_design, normalized, rcond=None) + trend_model = trend_design @ trend_coeff + trend_strength = float(np.nanstd(trend_model - np.nanmean(trend_model))) + except Exception: + trend_strength = np.nan + + curve_strength = np.nan + try: + centered_quadratic = x ** 2 - float(np.nanmean(x ** 2)) + structure_design = np.column_stack([ + np.ones(point_count, dtype=float), + x, + centered_quadratic, + np.sin(2.0 * np.pi * x01), + np.cos(2.0 * np.pi * x01), + np.sin(4.0 * np.pi * x01), + np.cos(4.0 * np.pi * x01), + ]) + structure_coeff, *_ = np.linalg.lstsq(structure_design, normalized, rcond=None) + structure_model = structure_design @ structure_coeff + curve_strength = float(np.nanstd(structure_model - np.nanmean(structure_model))) + except Exception: + curve_strength = np.nan + + binned_curve_strength = np.nan + scatter_ratio = np.nan + bin_count = int(np.clip(point_count // 8, 4, 8)) + bins = [ + chunk for chunk in np.array_split(np.arange(point_count), bin_count) + if chunk.size >= 3 + ] + if len(bins) >= 3: + bin_medians = np.asarray([ + float(np.nanmedian(normalized[chunk])) + for chunk in bins + ], dtype=float) + binned_curve_strength = float(np.nanstd(bin_medians - np.nanmedian(bin_medians))) + + bin_sigmas = np.asarray([ + robust_sigma(normalized[chunk] - float(np.nanmedian(normalized[chunk]))) + for chunk in bins + ], dtype=float) + finite_sigmas = bin_sigmas[np.isfinite(bin_sigmas) & (bin_sigmas > np.finfo(float).eps)] + if finite_sigmas.size >= 2: + median_sigma = float(np.nanmedian(finite_sigmas)) + if np.isfinite(median_sigma) and median_sigma > np.finfo(float).eps: + finite_sizes = np.asarray([ + chunk.size + for chunk, sigma in zip(bins, bin_sigmas) + if np.isfinite(sigma) and sigma > np.finfo(float).eps + ], dtype=float) + log_scatter_offsets = np.abs(np.log(finite_sigmas / median_sigma)) + expected_log_scatter_noise = 1.0 / np.sqrt(2.0 * np.maximum(finite_sizes - 1.0, 1.0)) + excess_log_scatter_offsets = np.maximum( + 0.0, + log_scatter_offsets - expected_log_scatter_noise, + ) + if excess_log_scatter_offsets.size: + scatter_ratio = float(np.exp(np.nanpercentile(excess_log_scatter_offsets, 80.0))) + + if np.isfinite(binned_curve_strength): + curve_strength = ( + max(curve_strength, binned_curve_strength) + if np.isfinite(curve_strength) + else binned_curve_strength + ) + + trend_score = transit_qc_flatness_declining_score(trend_strength, 0.20, 0.85) + curve_score = transit_qc_flatness_declining_score(curve_strength, 0.25, 1.00) + scatter_stability_score = ( + transit_qc_flatness_declining_score(np.log(scatter_ratio), np.log(1.5), np.log(3.0)) + if np.isfinite(scatter_ratio) and scatter_ratio > 0 + else np.nan + ) + zero_offset_score = transit_qc_flatness_declining_score(zero_offset_strength, 0.25, 1.25) + sign_balance_score = ( + transit_qc_flatness_declining_score(sign_imbalance, 0.35, 0.85) + if np.isfinite(sign_imbalance) + else np.nan + ) + zero_bias_score = np.nanmin([ + value for value in (zero_offset_score, sign_balance_score) + if np.isfinite(value) + ]) if np.isfinite(zero_offset_score) or np.isfinite(sign_balance_score) else np.nan + component_scores = { + 'zero bias': zero_bias_score, + 'trend': trend_score, + 'curvature/sinusoid': curve_score, + 'scatter stability': scatter_stability_score, + } + finite_component_scores = { + key: float(value) + for key, value in component_scores.items() + if np.isfinite(value) + } + if not finite_component_scores: + summary['detail'] = "residual flatness components unavailable" + return summary + + dominant = min(finite_component_scores, key=finite_component_scores.get) + score = finite_component_scores[dominant] + detail_parts = [ + f"{dominant} limited", + f"median offset={zero_offset_strength:.2f}" if np.isfinite(zero_offset_strength) else "median offset=n/a", + f"sign imbalance={sign_imbalance:.2f}" if np.isfinite(sign_imbalance) else "sign imbalance=n/a", + f"trend={trend_strength:.2f}" if np.isfinite(trend_strength) else "trend=n/a", + f"curve={curve_strength:.2f}" if np.isfinite(curve_strength) else "curve=n/a", + f"scatter ratio={scatter_ratio:.2f}" if np.isfinite(scatter_ratio) else "scatter ratio=n/a", + f"n={point_count}", + ] + summary.update({ + 'available': True, + 'score': float(np.clip(score, 0.0, 1.0)), + 'scatter': float(scatter), + 'trend_strength': trend_strength, + 'curve_strength': curve_strength, + 'scatter_ratio': scatter_ratio, + 'zero_offset_strength': zero_offset_strength, + 'sign_imbalance': sign_imbalance, + 'zero_bias_score': zero_bias_score, + 'trend_score': trend_score, + 'curve_score': curve_score, + 'scatter_stability_score': scatter_stability_score, + 'dominant': dominant, + 'detail': ", ".join(detail_parts), + }) + return summary + + +def transit_qc_mean_available_score(*scores): + finite_scores = [float(score) for score in scores if np.isfinite(score)] + if not finite_scores: + return np.nan + return float(np.clip(np.mean(finite_scores), 0.0, 1.0)) + + +def coerce_finite_transit_qc_scalar(value): + if value is None: + return np.nan + + if isinstance(value, (list, tuple, np.ndarray)): + array_value = np.asarray(value) + if array_value.size != 1: + return np.nan + value = array_value.reshape(-1)[0] + + try: + numeric_value = float(value.strip()) if isinstance(value, str) else float(value) + except (AttributeError, TypeError, ValueError): + return np.nan + + if not np.isfinite(numeric_value): + return np.nan + return float(numeric_value) + + +def fit_transit_qc_expected_context(fit): + if fit is None: + return {} + + return { + 'expected_tmid': coerce_finite_transit_qc_scalar( + getattr(fit, 'transit_qc_expected_tmid', np.nan) + ), + 'expected_tmid_unc': coerce_finite_transit_qc_scalar( + getattr(fit, 'transit_qc_expected_tmid_unc', np.nan) + ), + 'expected_rprs': coerce_finite_transit_qc_scalar( + getattr(fit, 'transit_qc_expected_rprs', np.nan) + ), + 'expected_rprs_unc': coerce_finite_transit_qc_scalar( + getattr(fit, 'transit_qc_expected_rprs_unc', np.nan) + ), + 'use_deviation_from_expected_transit_in_qc': should_use_deviation_from_expected_transit_in_qc( + getattr( + fit, + 'transit_qc_use_deviation_from_expected_transit_in_qc', + TRANSIT_QC_USE_DEVIATION_FROM_EXPECTED_DEFAULT, + ) + ), + 'deviation_sigma_threshold': parse_deviation_from_expected_transit_in_qc_sigma( + getattr( + fit, + 'transit_qc_deviation_sigma_threshold', + TRANSIT_QC_DEVIATION_SIGMA_DEFAULT, + ) + ), + } + + +def annotate_transit_qc_expected_values(fit, planet_dict): + if fit is None or not isinstance(planet_dict, dict): + return + + expected_tmid = coerce_finite_transit_qc_scalar( + getattr(fit, 'initial_tmid_search_tmid', np.nan) + ) + expected_tmid_unc = coerce_finite_transit_qc_scalar( + getattr(fit, 'initial_tmid_search_uncertainty', np.nan) + ) + if not np.isfinite(expected_tmid): + expected_tmid = coerce_finite_transit_qc_scalar(planet_dict.get('midT', np.nan)) + if not np.isfinite(expected_tmid_unc): + expected_tmid_unc = coerce_finite_transit_qc_scalar(planet_dict.get('midTUnc', np.nan)) + + fit.transit_qc_expected_tmid = expected_tmid + fit.transit_qc_expected_tmid_unc = expected_tmid_unc + fit.transit_qc_expected_rprs = coerce_finite_transit_qc_scalar( + planet_dict.get('rprs', np.nan) + ) + fit.transit_qc_expected_rprs_unc = coerce_finite_transit_qc_scalar( + planet_dict.get('rprsUnc', np.nan) + ) + fit.transit_qc_use_deviation_from_expected_transit_in_qc = should_use_deviation_from_expected_transit_in_qc( + planet_dict.get( + 'use_deviation_from_expected_transit_in_qc', + TRANSIT_QC_USE_DEVIATION_FROM_EXPECTED_DEFAULT, + ) + ) + fit.transit_qc_deviation_sigma_threshold = parse_deviation_from_expected_transit_in_qc_sigma( + planet_dict.get( + 'deviation_from_expected_transit_in_qc_sigma', + TRANSIT_QC_DEVIATION_SIGMA_DEFAULT, + ) + ) + + +def annotate_transit_qc_fit_context( + fit, + planet_dict=None, + tmid_search_summary=None, + eebls_search_summary=None, +): + if fit is None: + return + + if tmid_search_summary is not None: + annotate_lightcurve_tmid_search(fit, tmid_search_summary) + if eebls_search_summary is not None: + annotate_lightcurve_eebls_diagnostic(fit, eebls_search_summary) + annotate_transit_qc_expected_values(fit, planet_dict) + + +def copy_transit_qc_expected_values(source_fit, target_fit): + if source_fit is None or target_fit is None: + return + + for attr in ( + 'transit_qc_expected_tmid', + 'transit_qc_expected_tmid_unc', + 'transit_qc_expected_rprs', + 'transit_qc_expected_rprs_unc', + 'transit_qc_use_deviation_from_expected_transit_in_qc', + 'transit_qc_deviation_sigma_threshold', + ): + if hasattr(source_fit, attr): + setattr(target_fit, attr, getattr(source_fit, attr)) + + +def evaluate_transit_qc_expected_value_deviation(fit, sigma_threshold, enabled=True): + summary = { + 'enabled': bool(enabled), + 'sigma_threshold': sigma_threshold, + 'expected_tmid': np.nan, + 'expected_tmid_unc': np.nan, + 'expected_tmid_unc_minutes': np.nan, + 'fitted_tmid': np.nan, + 'fitted_rprs': np.nan, + 'fitted_rprs_unc': np.nan, + 'tmid_deviation_days': np.nan, + 'tmid_deviation_minutes': np.nan, + 'tmid_deviation_threshold_minutes': np.nan, + 'tmid_deviation_sigma': np.nan, + 'rprs_deviation_fit_unc': np.nan, + 'rprs_deviation_model_fit_unc': np.nan, + 'rprs_deviation_data_fit_unc': np.nan, + 'rprs_deviation_combined_fit_unc': np.nan, + 'rprs_deviation_expected_unc': np.nan, + 'rprs_deviation_systematic_floor': np.nan, + 'rprs_deviation_unc': np.nan, + 'rprs_deviation_sigma': np.nan, + 'tmid_deviation_score': np.nan, + 'rprs_deviation_score': np.nan, + 'deviation_from_expected_value': np.nan, + 'rprs_prior_assumed': False, + 'rprs_prior_assumed_note': None, + 'available': False, + 'failed': False, + 'notes': [], + 'failure_reasons': [], + } + if fit is None: + return summary + + parameters = getattr(fit, 'parameters', {}) or {} + expected = fit_transit_qc_expected_context(fit) + sigma_threshold = expected.get('deviation_sigma_threshold', sigma_threshold) + summary['sigma_threshold'] = sigma_threshold + + summary['expected_tmid'] = expected.get('expected_tmid', np.nan) + summary['expected_tmid_unc'] = expected.get('expected_tmid_unc', np.nan) + summary['fitted_tmid'] = parameters.get('tmid', np.nan) + if not enabled: + return summary + + expected_rprs = expected.get('expected_rprs', np.nan) + expected_rprs_unc = expected.get('expected_rprs_unc', np.nan) + fitted_rprs = parameters.get('rprs', np.nan) + errors = getattr(fit, 'errors', {}) or {} + fitted_rprs_model_unc = errors.get('rprs', np.nan) + empirical_uncertainty = getattr(fit, 'empirical_transit_uncertainty', None) + if not isinstance(empirical_uncertainty, dict) or not empirical_uncertainty.get('available'): + empirical_uncertainty = fit_empirical_transit_uncertainty(fit) + if isinstance(empirical_uncertainty, dict) and empirical_uncertainty.get('available'): + try: + fit.empirical_transit_uncertainty = empirical_uncertainty + except Exception: + pass + fitted_rprs_data_unc = np.nan + fitted_rprs_unc = fitted_rprs_model_unc + rprs_prior_assumed = False + if isinstance(empirical_uncertainty, dict) and empirical_uncertainty.get('available'): + rprs_prior_assumed = ( + bool(empirical_uncertainty.get('rprs_prior_fallback_applied')) + or empirical_uncertainty.get('rprs_uncertainty_basis') == 'prior_assumed_data_only' + ) + if rprs_prior_assumed: + fitted_rprs_model_unc = np.nan + fitted_rprs_data_unc = empirical_uncertainty.get('data_rprs_uncertainty', np.nan) + combined_uncertainty = _finite_float( + empirical_uncertainty.get('combined_rprs_uncertainty'), + np.nan, + ) + if np.isfinite(combined_uncertainty) and combined_uncertainty >= 0: + fitted_rprs_unc = combined_uncertainty + comparison_unc, systematic_floor = transit_qc_rprs_deviation_uncertainty( + fitted_rprs_unc, + expected_rprs_unc, + expected_rprs, + ) + summary['expected_rprs'] = expected_rprs + summary['expected_rprs_unc'] = expected_rprs_unc + summary['fitted_rprs'] = fitted_rprs + summary['fitted_rprs_unc'] = fitted_rprs_unc + summary['rprs_deviation_fit_unc'] = fitted_rprs_unc + summary['rprs_deviation_model_fit_unc'] = fitted_rprs_model_unc + summary['rprs_deviation_data_fit_unc'] = fitted_rprs_data_unc + summary['rprs_deviation_combined_fit_unc'] = fitted_rprs_unc + summary['rprs_deviation_expected_unc'] = expected_rprs_unc + summary['rprs_deviation_systematic_floor'] = systematic_floor + summary['rprs_deviation_unc'] = comparison_unc + summary['rprs_prior_assumed'] = bool(rprs_prior_assumed) + if rprs_prior_assumed: + summary['rprs_prior_assumed_note'] = ( + "Rp/R* was fixed to the input prior; the expected-value deviation is circular " + "and is omitted from KTMF scoring." + ) + if ( + np.isfinite(expected_rprs) + and np.isfinite(fitted_rprs) + and np.isfinite(comparison_unc) + and comparison_unc > 0 + ): + rprs_sigma = float(abs(fitted_rprs - expected_rprs) / comparison_unc) + summary['rprs_deviation_sigma'] = rprs_sigma + summary['rprs_deviation_score'] = transit_qc_deviation_score_from_sigma(rprs_sigma, sigma_threshold) + + if np.isfinite(summary['rprs_deviation_score']): + summary['available'] = True + summary['deviation_from_expected_value'] = float(summary['rprs_deviation_score']) + + if np.isfinite(summary['rprs_deviation_sigma']): + uncertainty_parts = [] + model_unc = summary.get('rprs_deviation_model_fit_unc', np.nan) + data_unc = summary.get('rprs_deviation_data_fit_unc', np.nan) + if np.isfinite(model_unc): + uncertainty_parts.append(f"model={model_unc:.6f}") + if np.isfinite(data_unc): + uncertainty_parts.append(f"data/red-noise={data_unc:.6f}") + uncertainty_note = ( + "; fit uncertainty terms: " + ", ".join(uncertainty_parts) + if uncertainty_parts + else "" + ) + summary['notes'].append( + "Expected-value Rp/R* deviation: " + f"{summary['rprs_deviation_sigma']:.2f} sigma " + f"(fit={summary['fitted_rprs']:.6f} +/- {summary['fitted_rprs_unc']:.6f}, " + f"expected={expected_rprs:.6f} +/- {expected_rprs_unc:.6f}; " + f"comparison uncertainty={summary['rprs_deviation_unc']:.6f}, " + f"including {100.0 * TRANSIT_QC_RPRS_DEVIATION_SYSTEMATIC_FLOOR_FRACTION:.1f}% " + f"Rp/R* floor={summary['rprs_deviation_systematic_floor']:.6f}" + f"{uncertainty_note})." + ) + if rprs_prior_assumed: + summary['notes'].append(summary['rprs_prior_assumed_note']) + + rprs_sigma = summary['rprs_deviation_sigma'] + if np.isfinite(rprs_sigma) and np.isfinite(sigma_threshold) and sigma_threshold > 0 and rprs_sigma > sigma_threshold: + summary['failed'] = True + reason = f"Rp/R* differs from the expected value by more than {sigma_threshold:.2f} sigma" + summary['notes'].append(reason + ".") + summary['failure_reasons'].append(reason) + + return summary + + +def _transit_qc_weighted_quantiles(values, probabilities, weights=None): + values = np.asarray(values, dtype=float).reshape(-1) + probabilities = np.asarray(probabilities, dtype=float) + finite_mask = np.isfinite(values) + + finite_weights = None + if weights is not None: + weights = np.asarray(weights, dtype=float).reshape(-1) + if weights.shape == values.shape: + finite_mask &= np.isfinite(weights) & (weights >= 0) + finite_weights = weights[finite_mask] + if finite_weights.size == 0 or np.sum(finite_weights) <= 0: + finite_weights = None + + finite_values = values[finite_mask] + if finite_values.size == 0: + return np.full(probabilities.shape, np.nan, dtype=float) + if finite_weights is None: + return np.nanpercentile(finite_values, 100.0 * probabilities) + + order = np.argsort(finite_values) + sorted_values = finite_values[order] + sorted_weights = finite_weights[order] + cumulative = np.cumsum(sorted_weights) + total = cumulative[-1] + if not np.isfinite(total) or total <= 0: + return np.nanpercentile(finite_values, 100.0 * probabilities) + + cumulative = (cumulative - 0.5 * sorted_weights) / total + cumulative = np.clip(cumulative, 0.0, 1.0) + return np.interp(probabilities, cumulative, sorted_values) + + +def _transit_qc_tmid_gaussianity_shape(values, weights=None): + probabilities = np.linspace(0.02, 0.98, 49) + posterior_quantiles = _transit_qc_weighted_quantiles(values, probabilities, weights) + q16, q50, q84 = _transit_qc_weighted_quantiles(values, [0.16, 0.50, 0.84], weights) + robust_sigma = float((q84 - q16) / 2.0) + minimum_scale = np.finfo(float).eps * max(1.0, abs(float(q50))) + if ( + not np.all(np.isfinite(posterior_quantiles)) + or not np.isfinite(q50) + or not np.isfinite(robust_sigma) + or robust_sigma <= minimum_scale + ): + return np.nan, np.nan, np.nan, robust_sigma + + standardized_quantiles = (posterior_quantiles - q50) / robust_sigma + gaussian_template = ndtri(probabilities) + # A uniform distribution has q84-q16 = 0.68 of its full width, so its + # robust sigma is 0.34 of that width after applying the same scaling. + flat_template = (probabilities - 0.5) / 0.34 + gaussian_distance = float(np.mean((standardized_quantiles - gaussian_template) ** 2)) + flat_distance = float(np.mean((standardized_quantiles - flat_template) ** 2)) + + if not np.isfinite(gaussian_distance) or not np.isfinite(flat_distance): + return np.nan, gaussian_distance, flat_distance, robust_sigma + if flat_distance <= np.finfo(float).eps: + score = 0.0 + else: + score = float(np.clip(1.0 - gaussian_distance / flat_distance, 0.0, 1.0)) + return score, gaussian_distance, flat_distance, robust_sigma + + +def transit_qc_tmid_gaussianity_summary(fit): + summary = { + 'available': False, + 'score': np.nan, + 'score_uncertainty': np.nan, + 'gaussian_distance': np.nan, + 'flat_distance': np.nan, + 'effective_sample_count': 0.0, + 'sample_count': 0, + 'robust_sigma': np.nan, + 'detail': 'Tmid posterior Gaussianity is unavailable.', + } + if fit is None: + return summary + + sampled_keys = getattr(fit, 'sampled_keys', None) + if sampled_keys is not None and 'tmid' not in list(sampled_keys): + summary['detail'] = 'Tmid was fixed rather than sampled; posterior Gaussianity is not scored.' + return summary + + sample_matrix, sample_weights = _fit_posterior_sample_matrix(fit, ['tmid']) + if sample_matrix.size == 0 or sample_matrix.shape[0] == 0: + summary['detail'] = 'Weighted Tmid posterior samples are unavailable.' + return summary + + values = np.asarray(sample_matrix[:, 0], dtype=float).reshape(-1) + finite_mask = np.isfinite(values) + parameter_weights = None + if sample_weights is not None: + sample_weights = np.asarray(sample_weights, dtype=float).reshape(-1) + if sample_weights.shape == values.shape: + finite_mask &= np.isfinite(sample_weights) & (sample_weights >= 0) + parameter_weights = sample_weights[finite_mask] + if parameter_weights.size == 0 or np.sum(parameter_weights) <= 0: + parameter_weights = None + + values = values[finite_mask] + sample_count = int(values.size) + effective_sample_count = _effective_sample_count(parameter_weights, sample_count) + summary['sample_count'] = sample_count + summary['effective_sample_count'] = float(effective_sample_count) + if sample_count < 2: + summary['detail'] = 'Too few finite Tmid posterior samples to assess Gaussianity.' + return summary + if effective_sample_count < TRANSIT_QC_TMID_GAUSSIANITY_MIN_EFFECTIVE_SAMPLES: + summary['detail'] = ( + 'Tmid posterior Gaussianity is not scored because effective samples ' + f'{effective_sample_count:.0f} < {TRANSIT_QC_TMID_GAUSSIANITY_MIN_EFFECTIVE_SAMPLES}.' + ) + return summary + + score, gaussian_distance, flat_distance, robust_sigma = _transit_qc_tmid_gaussianity_shape( + values, + parameter_weights, + ) + summary.update({ + 'score': score, + 'gaussian_distance': gaussian_distance, + 'flat_distance': flat_distance, + 'robust_sigma': robust_sigma, + }) + if not np.isfinite(score): + summary['detail'] = 'Tmid posterior Gaussianity is unavailable because its robust width is degenerate.' + return summary + + bootstrap_size = int(np.clip( + round(effective_sample_count), + TRANSIT_QC_TMID_GAUSSIANITY_MIN_EFFECTIVE_SAMPLES, + TRANSIT_QC_TMID_GAUSSIANITY_BOOTSTRAP_MAX_SAMPLES, + )) + choice_probabilities = None + if parameter_weights is not None: + choice_probabilities = parameter_weights / np.sum(parameter_weights) + rng = np.random.default_rng(24601) + bootstrap_indices = rng.choice( + sample_count, + size=(TRANSIT_QC_TMID_GAUSSIANITY_BOOTSTRAP_DRAWS, bootstrap_size), + replace=True, + p=choice_probabilities, + ) + bootstrap_scores = [] + for indices in bootstrap_indices: + bootstrap_score, _, _, _ = _transit_qc_tmid_gaussianity_shape(values[indices]) + if np.isfinite(bootstrap_score): + bootstrap_scores.append(float(bootstrap_score)) + if len(bootstrap_scores) > 1: + summary['score_uncertainty'] = float(np.std(bootstrap_scores, ddof=1)) + + if score >= 0.85: + interpretation = 'strongly Gaussian-like' + elif score >= 0.60: + interpretation = 'broadly Gaussian-like' + elif score >= 0.20: + interpretation = 'weakly Gaussian-like' + else: + interpretation = 'flat-like or strongly non-Gaussian' + uncertainty = summary['score_uncertainty'] + uncertainty_text = f' +/- {uncertainty:.2f}' if np.isfinite(uncertainty) else '' + summary.update({ + 'available': True, + 'detail': ( + f'{interpretation}; weighted-quantile score={score:.2f}{uncertainty_text}, ' + f'effective samples={effective_sample_count:.0f}, ' + f'Gaussian mismatch={gaussian_distance:.4f}, flat mismatch={flat_distance:.4f}' + ), + }) + return summary + + +def compute_transit_qc_ktmf(summary): + if not isinstance(summary, dict): + return np.nan, [] + + deviation_score = summary.get('deviation_from_expected_value', np.nan) + rprs_prior_assumed = bool(summary.get('rprs_prior_assumed', False)) + geometry_prior_assumed = bool(summary.get('geometry_prior_assumed', False)) + if rprs_prior_assumed: + deviation_score = np.nan + deviation_detail = ( + summary.get('rprs_prior_assumed_note') + or "Rp/R* was fixed to the input prior; expected-value deviation is omitted from KTMF scoring." + ) + elif np.isfinite(deviation_score): + deviation_detail_parts = [ + f"score={deviation_score:.2f}", + f"Rp/R* sigma={summary.get('rprs_deviation_sigma', np.nan):.2f}", + ] + rprs_fit_unc = summary.get('rprs_deviation_fit_unc', np.nan) + if np.isfinite(rprs_fit_unc): + deviation_detail_parts.append(f"fit uncertainty={rprs_fit_unc:.6f}") + model_fit_unc = summary.get('rprs_deviation_model_fit_unc', np.nan) + if np.isfinite(model_fit_unc): + deviation_detail_parts.append(f"model uncertainty={model_fit_unc:.6f}") + data_fit_unc = summary.get('rprs_deviation_data_fit_unc', np.nan) + if np.isfinite(data_fit_unc): + deviation_detail_parts.append(f"data/red-noise uncertainty={data_fit_unc:.6f}") + expected_unc = summary.get('rprs_deviation_expected_unc', summary.get('expected_rprs_unc', np.nan)) + if np.isfinite(expected_unc): + deviation_detail_parts.append(f"expected uncertainty={expected_unc:.6f}") + comparison_unc = summary.get('rprs_deviation_unc', np.nan) + if np.isfinite(comparison_unc): + deviation_detail_parts.append(f"comparison uncertainty={comparison_unc:.6f}") + systematic_floor = summary.get('rprs_deviation_systematic_floor', np.nan) + if np.isfinite(systematic_floor): + deviation_detail_parts.append(f"systematic floor={systematic_floor:.6f}") + deviation_detail = ", ".join(deviation_detail_parts) + else: + deviation_detail = "expected-value deviation disabled or unavailable" + + if geometry_prior_assumed: + geometry_note = ( + summary.get('geometry_prior_assumed_note') + or "Transit geometry was fixed to input priors for partial-coverage fitting." + ) + deviation_score = np.nan + deviation_detail = ( + f"{geometry_note} Expected-value deviation is omitted from KTMF scoring." + ) + + residual_scatter = summary.get('residual_scatter', np.nan) + residual_depth = summary.get('transit_depth_for_residual_scatter', np.nan) + residual_scatter_to_depth_ratio = summary.get('residual_scatter_to_depth_ratio', np.nan) + residual_scatter_score = transit_qc_residual_scatter_score( + residual_scatter, + residual_depth, + ) + residual_flatness_score = summary.get('residual_flatness_score', np.nan) + residual_flatness_detail = summary.get('residual_flatness_detail') or "n/a" + tmid_gaussianity_score = summary.get('tmid_gaussianity_score', np.nan) + tmid_gaussianity_score_uncertainty = summary.get('tmid_gaussianity_score_uncertainty', np.nan) + tmid_gaussianity_detail = ( + summary.get('tmid_gaussianity_detail') + or "Tmid posterior Gaussianity is unavailable." + ) + residual_scatter_score_uncertainty = np.nan + point_count = summary.get('point_count', np.nan) + if ( + np.isfinite(residual_scatter) + and residual_scatter >= 0 + and np.isfinite(residual_depth) + and residual_depth > 0 + and np.isfinite(point_count) + and point_count > 2 + ): + residual_scatter_uncertainty = residual_scatter / np.sqrt(2.0 * (point_count - 1.0)) + residual_ratio_uncertainty = residual_scatter_uncertainty / residual_depth + full_credit_ratio = 0.5 + zero_credit_ratio = 4.0 + decay_rate = 3.0 + if not np.isfinite(residual_scatter_to_depth_ratio): + residual_scatter_to_depth_ratio = residual_scatter / residual_depth + if residual_scatter_to_depth_ratio <= full_credit_ratio or residual_scatter_to_depth_ratio >= zero_credit_ratio: + residual_scatter_score_uncertainty = 0.0 + else: + interval_fraction = ( + (residual_scatter_to_depth_ratio - full_credit_ratio) + / (zero_credit_ratio - full_credit_ratio) + ) + derivative = ( + decay_rate * np.exp(-decay_rate * interval_fraction) + / ((zero_credit_ratio - full_credit_ratio) * (1.0 - np.exp(-decay_rate))) + ) + residual_scatter_score_uncertainty = float(derivative * residual_ratio_uncertainty) + + if ( + np.isfinite(residual_scatter) + and np.isfinite(residual_depth) + and residual_depth > 0 + ): + residual_scatter_detail = ( + f"scatter/depth={residual_scatter / residual_depth:.2f}, " + f"scatter={residual_scatter * 100.0:.4f}%, " + f"depth={residual_depth * 100.0:.3f}%" + ) + elif np.isfinite(residual_scatter): + residual_scatter_detail = f"scatter={residual_scatter * 100.0:.4f}%, depth=n/a" + else: + residual_scatter_detail = "n/a" + + duration_note = summary.get('duration_consistency_note') + if np.isfinite(summary.get('duration_ratio', np.nan)): + duration_detail = f"{summary.get('duration_ratio', np.nan):.2f}x expected duration" + if duration_note: + duration_detail = f"{duration_detail}; {duration_note}" + else: + duration_detail = duration_note or "n/a" + + duration_score = transit_qc_duration_score(summary.get('duration_ratio', np.nan)) + sampling_score = summary.get('sampling_score', np.nan) + sampling_detail = summary.get('sampling_detail') or "n/a" + if geometry_prior_assumed: + geometry_note = ( + summary.get('geometry_prior_assumed_note') + or "Transit geometry was fixed to input priors for partial-coverage fitting." + ) + duration_score = np.nan + duration_detail = ( + f"{geometry_note} Duration consistency is omitted from KTMF scoring." + ) + sampling_score = np.nan + sampling_detail = ( + f"{geometry_note} Sampling/cadence is omitted from KTMF scoring." + ) + + raw_components = [ + { + 'key': 'deviation_from_expected_value', + 'label': 'Deviation From Expected Value', + 'score': deviation_score, + 'score_uncertainty': np.nan, + 'detail': deviation_detail, + }, + { + 'key': 'residual_scatter', + 'label': 'Residual Scatter Around Full Model Fit', + 'score': residual_scatter_score, + 'score_uncertainty': residual_scatter_score_uncertainty, + 'detail': residual_scatter_detail, + }, + { + 'key': 'residual_flatness', + 'label': 'Residual Flatness', + 'score': residual_flatness_score, + 'score_uncertainty': np.nan, + 'detail': residual_flatness_detail, + }, + { + 'key': 'tmid_gaussianity', + 'label': 'Tmid Posterior Gaussianity', + 'score': tmid_gaussianity_score, + 'score_uncertainty': tmid_gaussianity_score_uncertainty, + 'detail': tmid_gaussianity_detail, + }, + { + 'key': 'duration_consistency', + 'label': 'Duration Consistency', + 'score': duration_score, + 'score_uncertainty': np.nan, + 'detail': duration_detail, + }, + { + 'key': 'eebls_depth_snr', + 'label': 'EEBLS Depth SNR', + 'score': transit_qc_saturating_score(summary.get('eebls_depth_snr', np.nan), TRANSIT_QC_MIN_EEBLS_SNR), + 'score_uncertainty': np.nan, + 'detail': ( + f"{summary.get('eebls_depth_snr', np.nan):.2f}" + if np.isfinite(summary.get('eebls_depth_snr', np.nan)) + else "n/a" + ), + }, + { + 'key': 'sampling', + 'label': 'Sampling / Cadence', + 'score': sampling_score, + 'score_uncertainty': np.nan, + 'detail': sampling_detail, + }, + ] + + available_components = [ + component + for component in raw_components + if np.isfinite(component.get('score', np.nan)) + and component['key'] in TRANSIT_QC_KTMF_COMPONENT_MAX_POINTS + ] + if not available_components: + return np.nan, [] + + available_max_points = sum(TRANSIT_QC_KTMF_COMPONENT_MAX_POINTS[component['key']] for component in available_components) + if not np.isfinite(available_max_points) or available_max_points <= 0: + return np.nan, [] + + scale_factor = 5.0 / available_max_points + ktmf_contributions = [] + total_points = 0.0 + for component in raw_components: + nominal_max_points = TRANSIT_QC_KTMF_COMPONENT_MAX_POINTS.get(component['key'], 0.0) + score = component.get('score', np.nan) + if np.isfinite(score) and nominal_max_points > 0: + max_points = nominal_max_points * scale_factor + points = float(np.clip(score, 0.0, 1.0) * max_points) + total_points += points + ktmf_contributions.append({ + 'key': component['key'], + 'label': component['label'], + 'score': float(np.clip(score, 0.0, 1.0)), + 'score_uncertainty': component.get('score_uncertainty', np.nan), + 'max_points': float(max_points), + 'points': points, + 'detail': component.get('detail'), + 'available': True, + }) + else: + ktmf_contributions.append({ + 'key': component['key'], + 'label': component['label'], + 'score': np.nan, + 'score_uncertainty': np.nan, + 'max_points': 0.0, + 'points': 0.0, + 'detail': component.get('detail'), + 'available': False, + }) + + return float(np.clip(total_points, 0.0, 5.0)), ktmf_contributions + + +def infer_transit_qc_parameter_count(fit, allow_airmass_term): + bounds = getattr(fit, 'bounds', None) + if isinstance(bounds, dict) and bounds: + parameter_count = len(bounds) + if 'a0' not in bounds and 'a1' not in bounds: + parameter_count += 1 + return max(int(parameter_count), 1) + + parameters = getattr(fit, 'parameters', {}) or {} + errors = getattr(fit, 'errors', {}) or {} + available_keys = set(parameters.keys()) | set(errors.keys()) + + parameter_count = 1 # profiled flux baseline + if 'rprs' in available_keys: + parameter_count += 1 + if 'tmid' in available_keys: + parameter_count += 1 + if 'inc' in available_keys or 'b' in available_keys: + parameter_count += 1 + if allow_airmass_term and 'a2' in available_keys: + parameter_count += 1 + return max(parameter_count, 1) + + +def fit_profiled_flat_null_model(data, dataerr, airmass, initial_a2=0.0, a2_bounds=None, allow_airmass_term=True): + data = np.asarray(data, dtype=float) + dataerr = None if dataerr is None else np.asarray(dataerr, dtype=float) + airmass = None if airmass is None else np.asarray(airmass, dtype=float) + + result = { + 'available': False, + 'used_airmass_term': False, + 'baseline': np.nan, + 'a2': 0.0, + 'model': np.full(data.shape, np.nan, dtype=float), + 'chi2': np.nan, + 'bic': np.nan, + 'point_count': 0, + 'param_count': 1, + 'note': 'Flat/null model was not evaluated.', + } + + if data.ndim != 1 or data.size == 0: + result['note'] = 'Flat/null model comparison unavailable: no 1D lightcurve data were provided.' + return result + + if dataerr is not None and dataerr.shape != data.shape: + dataerr = None + if airmass is not None and airmass.shape != data.shape: + airmass = None + + use_airmass_term = bool( + allow_airmass_term + and airmass is not None + and airmass.ndim == 1 + and airmass.shape == data.shape + and not should_skip_airmass_fit(airmass) + ) + result['used_airmass_term'] = use_airmass_term + result['param_count'] = 1 + int(use_airmass_term) + + if not use_airmass_term: + baseline = solve_transit_qc_flux_baseline(np.ones_like(data, dtype=float), data, dataerr) + if not np.isfinite(baseline): + result['note'] = 'Flat/null model comparison unavailable: could not solve the baseline flux level.' + return result + model = np.full(data.shape, baseline, dtype=float) + chi2, point_count = compute_transit_qc_model_chi2(data, model, dataerr) + result.update({ + 'available': np.isfinite(chi2), + 'baseline': float(baseline), + 'model': model, + 'chi2': chi2, + 'point_count': point_count, + 'bic': compute_transit_qc_bic(chi2, point_count, result['param_count']), + 'note': 'Compared against a flat baseline-only null model.', + }) + return result + + lower, upper = TRANSIT_QC_DEFAULT_A2_BOUNDS + if a2_bounds is not None: + try: + lower, upper = np.asarray(a2_bounds, dtype=float).reshape(-1)[:2] + except (TypeError, ValueError, IndexError): + lower, upper = TRANSIT_QC_DEFAULT_A2_BOUNDS + if not np.isfinite(lower) or not np.isfinite(upper) or lower >= upper: + lower, upper = TRANSIT_QC_DEFAULT_A2_BOUNDS + + if not np.isfinite(initial_a2): + initial_a2 = 0.0 + initial_a2 = float(np.clip(initial_a2, lower + np.finfo(float).eps, upper - np.finfo(float).eps)) + reference = transit_qc_airmass_reference(airmass) + + def build_model(a2_value): + systematics = transit_qc_airmass_trend(a2_value, airmass, reference=reference) + baseline = solve_transit_qc_flux_baseline(systematics, data, dataerr) + if not np.isfinite(baseline): + return np.full(data.shape, np.nan, dtype=float), np.nan + return baseline * systematics, baseline + + def residual_vector(params): + model, baseline = build_model(params[0]) + if not np.isfinite(baseline): + return np.full(max(1, data.size), 1e6, dtype=float) + + mask = np.isfinite(data) & np.isfinite(model) + if dataerr is not None: + mask &= np.isfinite(dataerr) & (dataerr > 0) + if not np.any(mask): + return np.full(max(1, data.size), 1e6, dtype=float) + + if dataerr is not None: + return (data[mask] - model[mask]) / dataerr[mask] + return data[mask] - model[mask] + + best_a2 = float(initial_a2) + try: + fit_result = least_squares( + residual_vector, + x0=np.array([initial_a2], dtype=float), + bounds=([lower], [upper]), + ) + if fit_result.x.size: + best_a2 = float(fit_result.x[0]) + except Exception: + pass + + model, baseline = build_model(best_a2) + if not np.isfinite(baseline): + result['note'] = 'Flat/null model comparison unavailable: the null-model fit did not converge.' + return result + + chi2, point_count = compute_transit_qc_model_chi2(data, model, dataerr) + result.update({ + 'available': np.isfinite(chi2), + 'baseline': float(baseline), + 'a2': float(best_a2), + 'model': model, + 'chi2': chi2, + 'point_count': point_count, + 'bic': compute_transit_qc_bic(chi2, point_count, result['param_count']), + 'note': 'Compared against a flat null model with the same profiled baseline and airmass trend.', + }) + return result + + +def evaluate_transit_detection_qc(fit): + expected_context = fit_transit_qc_expected_context(fit) + use_deviation_from_expected_transit_in_qc = expected_context.get( + 'use_deviation_from_expected_transit_in_qc', + TRANSIT_QC_USE_DEVIATION_FROM_EXPECTED_DEFAULT, + ) + deviation_sigma_threshold = expected_context.get( + 'deviation_sigma_threshold', + TRANSIT_QC_DEVIATION_SIGMA_DEFAULT, + ) + summary = { + 'computed': False, + 'status': 'unknown', + 'preferred_model': 'unknown', + 'summary': 'Transit QC unavailable: no fit result was provided.', + 'notes': [], + 'transit_chi2': np.nan, + 'flat_chi2': np.nan, + 'delta_chi2': np.nan, + 'transit_bic': np.nan, + 'flat_bic': np.nan, + 'delta_bic': np.nan, + 'transit_parameter_count': 0, + 'flat_parameter_count': 0, + 'flat_baseline': np.nan, + 'flat_a2': np.nan, + 'flat_model_note': None, + 'rprs_sigma': np.nan, + 'duration_ratio': np.nan, + 'duration_measured_for_qc': np.nan, + 'duration_consistency_applicable': True, + 'duration_consistency_note': None, + 'eebls_depth_snr': np.nan, + 'residual_scatter': np.nan, + 'transit_depth_for_residual_scatter': np.nan, + 'residual_scatter_to_depth_ratio': np.nan, + 'residual_flatness_score': np.nan, + 'residual_flatness_trend_strength': np.nan, + 'residual_flatness_curve_strength': np.nan, + 'residual_flatness_scatter_ratio': np.nan, + 'residual_flatness_zero_offset_strength': np.nan, + 'residual_flatness_sign_imbalance': np.nan, + 'residual_flatness_zero_bias_score': np.nan, + 'residual_flatness_trend_score': np.nan, + 'residual_flatness_curve_score': np.nan, + 'residual_flatness_scatter_stability_score': np.nan, + 'residual_flatness_dominant_metric': None, + 'residual_flatness_detail': None, + 'tmid_gaussianity_score': np.nan, + 'tmid_gaussianity_score_uncertainty': np.nan, + 'tmid_gaussianity_gaussian_distance': np.nan, + 'tmid_gaussianity_flat_distance': np.nan, + 'tmid_gaussianity_effective_sample_count': 0.0, + 'tmid_gaussianity_sample_count': 0, + 'tmid_gaussianity_robust_sigma': np.nan, + 'tmid_gaussianity_detail': None, + 'sampling_score': np.nan, + 'sampling_detail': None, + 'sampling_ingress_count': 0, + 'sampling_egress_count': 0, + 'sampling_in_transit_count': 0, + 'sampling_pre_baseline_count': 0, + 'sampling_post_baseline_count': 0, + 'sampling_total_duration': np.nan, + 'sampling_ingress_duration': np.nan, + 'use_deviation_from_expected_transit_in_qc': bool(use_deviation_from_expected_transit_in_qc), + 'deviation_sigma_threshold': deviation_sigma_threshold, + 'expected_tmid': expected_context.get('expected_tmid', np.nan), + 'expected_tmid_unc': expected_context.get('expected_tmid_unc', np.nan), + 'expected_tmid_unc_minutes': np.nan, + 'fitted_tmid': np.nan, + 'expected_rprs': expected_context.get('expected_rprs', np.nan), + 'expected_rprs_unc': expected_context.get('expected_rprs_unc', np.nan), + 'fitted_rprs': np.nan, + 'fitted_rprs_unc': np.nan, + 'tmid_deviation_days': np.nan, + 'tmid_deviation_minutes': np.nan, + 'tmid_deviation_threshold_minutes': np.nan, + 'tmid_deviation_sigma': np.nan, + 'rprs_deviation_fit_unc': np.nan, + 'rprs_deviation_model_fit_unc': np.nan, + 'rprs_deviation_data_fit_unc': np.nan, + 'rprs_deviation_combined_fit_unc': np.nan, + 'rprs_deviation_expected_unc': np.nan, + 'rprs_deviation_systematic_floor': np.nan, + 'rprs_deviation_unc': np.nan, + 'rprs_deviation_sigma': np.nan, + 'tmid_deviation_score': np.nan, + 'rprs_deviation_score': np.nan, + 'deviation_from_expected_value': np.nan, + 'ktmf_metric': np.nan, + 'ktmf_contributions': [], + 'point_count': 0, + 'geometry_prior_assumed': False, + 'geometry_prior_assumed_note': None, + } + if fit is None: + return summary + + data = np.asarray(getattr(fit, 'data', np.array([])), dtype=float) + if data.ndim != 1 or data.size == 0: + summary['summary'] = ( + "Transit QC unavailable: fit results do not expose the 1D lightcurve data needed for " + "a transit-vs-flat comparison." + ) + return summary + + dataerr_obj = getattr(fit, 'dataerr', None) + dataerr = None if dataerr_obj is None else np.asarray(dataerr_obj, dtype=float) + if dataerr is not None and dataerr.shape != data.shape: + dataerr = None + + transit_model_obj = getattr(fit, 'model', None) + if transit_model_obj is None and hasattr(fit, 'residuals'): + residuals = np.asarray(getattr(fit, 'residuals'), dtype=float) + if residuals.shape == data.shape: + transit_model_obj = data - residuals + if transit_model_obj is None: + summary['summary'] = ( + "Transit QC unavailable: fit results do not expose the modeled transit lightcurve needed " + "for a transit-vs-flat comparison." + ) + return summary + + transit_model = np.asarray(transit_model_obj, dtype=float) + if transit_model.shape != data.shape: + summary['summary'] = ( + "Transit QC unavailable: the fitted transit model shape does not match the lightcurve data." + ) + return summary + + airmass_obj = getattr(fit, 'airmass', None) + airmass = None if airmass_obj is None else np.asarray(airmass_obj, dtype=float) + if airmass is not None and airmass.shape != data.shape: + airmass = None + + allow_airmass_term = bool( + airmass is not None + and airmass.ndim == 1 + and not getattr(fit, 'airmass_fit_skipped', False) + ) + bounds = getattr(fit, 'bounds', None) + a2_bounds = bounds.get('a2') if isinstance(bounds, dict) else None + parameters = getattr(fit, 'parameters', {}) or {} + errors = getattr(fit, 'errors', {}) or {} + geometry_prior_assumed = bool( + getattr(fit, 'partial_transit_geometry_prior_assumption_applied', False) + ) + geometry_prior_assumed_note = getattr( + fit, + 'partial_transit_geometry_prior_assumption_note', + None, + ) + summary['geometry_prior_assumed'] = geometry_prior_assumed + summary['geometry_prior_assumed_note'] = geometry_prior_assumed_note + initial_a2 = parameters.get('a2', 0.0) + transit_depth_model_obj = getattr(fit, 'transit', None) + transit_depth_model = None + if transit_depth_model_obj is not None: + transit_depth_model = np.asarray(transit_depth_model_obj, dtype=float) + if transit_depth_model.shape != data.shape: + transit_depth_model = None + if transit_depth_model is None: + transit_depth_model = transit_model + + flat_model = fit_profiled_flat_null_model( + data, + dataerr, + airmass, + initial_a2=initial_a2, + a2_bounds=a2_bounds, + allow_airmass_term=allow_airmass_term, + ) + transit_chi2, point_count = compute_transit_qc_model_chi2(data, transit_model, dataerr) + transit_parameter_count = infer_transit_qc_parameter_count(fit, flat_model.get('used_airmass_term', False)) + transit_bic = compute_transit_qc_bic(transit_chi2, point_count, transit_parameter_count) + flat_bic = flat_model.get('bic', np.nan) + delta_chi2 = flat_model.get('chi2', np.nan) - transit_chi2 + delta_bic = flat_bic - transit_bic + + summary.update({ + 'computed': bool(bool(flat_model.get('available')) and np.isfinite(transit_chi2)), + 'transit_chi2': transit_chi2, + 'flat_chi2': flat_model.get('chi2', np.nan), + 'delta_chi2': delta_chi2, + 'transit_bic': transit_bic, + 'flat_bic': flat_bic, + 'delta_bic': delta_bic, + 'transit_parameter_count': int(transit_parameter_count), + 'flat_parameter_count': int(flat_model.get('param_count', 0)), + 'flat_baseline': flat_model.get('baseline', np.nan), + 'flat_a2': flat_model.get('a2', np.nan), + 'flat_model_note': flat_model.get('note'), + 'residual_scatter': transit_qc_residual_scatter(data, transit_model), + 'transit_depth_for_residual_scatter': transit_qc_model_depth_fraction(transit_depth_model), + 'point_count': int(point_count), + }) + residual_coordinates = getattr(fit, 'phase', None) + if residual_coordinates is None or np.asarray(residual_coordinates).shape != data.shape: + residual_coordinates = getattr(fit, 'time', None) + residual_flatness = transit_qc_residual_flatness_summary( + data - transit_model, + coordinates=residual_coordinates, + ) + summary.update({ + 'residual_flatness_score': residual_flatness.get('score', np.nan), + 'residual_flatness_trend_strength': residual_flatness.get('trend_strength', np.nan), + 'residual_flatness_curve_strength': residual_flatness.get('curve_strength', np.nan), + 'residual_flatness_scatter_ratio': residual_flatness.get('scatter_ratio', np.nan), + 'residual_flatness_zero_offset_strength': residual_flatness.get('zero_offset_strength', np.nan), + 'residual_flatness_sign_imbalance': residual_flatness.get('sign_imbalance', np.nan), + 'residual_flatness_zero_bias_score': residual_flatness.get('zero_bias_score', np.nan), + 'residual_flatness_trend_score': residual_flatness.get('trend_score', np.nan), + 'residual_flatness_curve_score': residual_flatness.get('curve_score', np.nan), + 'residual_flatness_scatter_stability_score': residual_flatness.get('scatter_stability_score', np.nan), + 'residual_flatness_dominant_metric': residual_flatness.get('dominant'), + 'residual_flatness_detail': residual_flatness.get('detail'), + }) + if ( + np.isfinite(summary['residual_scatter']) + and np.isfinite(summary['transit_depth_for_residual_scatter']) + and summary['transit_depth_for_residual_scatter'] > 0 + ): + summary['residual_scatter_to_depth_ratio'] = float( + summary['residual_scatter'] / summary['transit_depth_for_residual_scatter'] + ) + + if not summary['computed']: + note = flat_model.get('note') or 'flat/null model comparison failed.' + summary['summary'] = f"Transit QC unavailable: {note}" + return summary + + if np.isfinite(delta_chi2): + if delta_chi2 > 1e-12: + summary['preferred_model'] = 'transit' + elif delta_chi2 < -1e-12: + summary['preferred_model'] = 'flat' + else: + summary['preferred_model'] = 'ambiguous' + + rprs = parameters.get('rprs', np.nan) + rprs_err = errors.get('rprs', np.nan) + if np.isfinite(rprs) and np.isfinite(rprs_err) and rprs_err > 0: + summary['rprs_sigma'] = float(abs(rprs) / rprs_err) + + duration_expected = getattr(fit, 'duration_expected', np.nan) + duration_measured, duration_applicable, duration_note = transit_qc_duration_consistency_measurement(fit) + summary['duration_consistency_applicable'] = bool(duration_applicable) + summary['duration_consistency_note'] = duration_note + summary['duration_measured_for_qc'] = duration_measured + if ( + duration_applicable + and np.isfinite(duration_expected) + and duration_expected > 0 + and np.isfinite(duration_measured) + and duration_measured >= 0 + ): + summary['duration_ratio'] = float(duration_measured / duration_expected) + + ensure_lightcurve_fit_eebls_diagnostic(fit) + summary['eebls_depth_snr'] = extract_lightcurve_fit_eebls_snr(fit) + sampling_summary = transit_qc_sampling_summary(fit) + summary.update({ + 'sampling_score': sampling_summary.get('score', np.nan), + 'sampling_detail': sampling_summary.get('detail'), + 'sampling_ingress_count': sampling_summary.get('ingress_count', 0), + 'sampling_egress_count': sampling_summary.get('egress_count', 0), + 'sampling_in_transit_count': sampling_summary.get('in_transit_count', 0), + 'sampling_pre_baseline_count': sampling_summary.get('pre_baseline_count', 0), + 'sampling_post_baseline_count': sampling_summary.get('post_baseline_count', 0), + 'sampling_total_duration': sampling_summary.get('total_duration', np.nan), + 'sampling_ingress_duration': sampling_summary.get('ingress_duration', np.nan), + }) + tmid_gaussianity = transit_qc_tmid_gaussianity_summary(fit) + summary.update({ + 'tmid_gaussianity_score': tmid_gaussianity.get('score', np.nan), + 'tmid_gaussianity_score_uncertainty': tmid_gaussianity.get('score_uncertainty', np.nan), + 'tmid_gaussianity_gaussian_distance': tmid_gaussianity.get('gaussian_distance', np.nan), + 'tmid_gaussianity_flat_distance': tmid_gaussianity.get('flat_distance', np.nan), + 'tmid_gaussianity_effective_sample_count': tmid_gaussianity.get('effective_sample_count', 0.0), + 'tmid_gaussianity_sample_count': tmid_gaussianity.get('sample_count', 0), + 'tmid_gaussianity_robust_sigma': tmid_gaussianity.get('robust_sigma', np.nan), + 'tmid_gaussianity_detail': tmid_gaussianity.get('detail'), + }) + deviation_summary = evaluate_transit_qc_expected_value_deviation( + fit, + deviation_sigma_threshold, + enabled=use_deviation_from_expected_transit_in_qc, + ) + summary.update({ + 'expected_tmid': deviation_summary.get('expected_tmid', np.nan), + 'expected_tmid_unc': deviation_summary.get('expected_tmid_unc', np.nan), + 'expected_tmid_unc_minutes': deviation_summary.get('expected_tmid_unc_minutes', np.nan), + 'fitted_tmid': deviation_summary.get('fitted_tmid', np.nan), + 'fitted_rprs': deviation_summary.get('fitted_rprs', np.nan), + 'fitted_rprs_unc': deviation_summary.get('fitted_rprs_unc', np.nan), + 'tmid_deviation_days': deviation_summary.get('tmid_deviation_days', np.nan), + 'tmid_deviation_minutes': deviation_summary.get('tmid_deviation_minutes', np.nan), + 'tmid_deviation_threshold_minutes': deviation_summary.get('tmid_deviation_threshold_minutes', np.nan), + 'tmid_deviation_sigma': deviation_summary.get('tmid_deviation_sigma', np.nan), + 'rprs_deviation_fit_unc': deviation_summary.get('rprs_deviation_fit_unc', np.nan), + 'rprs_deviation_model_fit_unc': deviation_summary.get('rprs_deviation_model_fit_unc', np.nan), + 'rprs_deviation_data_fit_unc': deviation_summary.get('rprs_deviation_data_fit_unc', np.nan), + 'rprs_deviation_combined_fit_unc': deviation_summary.get('rprs_deviation_combined_fit_unc', np.nan), + 'rprs_deviation_expected_unc': deviation_summary.get('rprs_deviation_expected_unc', np.nan), + 'rprs_deviation_systematic_floor': deviation_summary.get('rprs_deviation_systematic_floor', np.nan), + 'rprs_deviation_unc': deviation_summary.get('rprs_deviation_unc', np.nan), + 'rprs_deviation_sigma': deviation_summary.get('rprs_deviation_sigma', np.nan), + 'tmid_deviation_score': deviation_summary.get('tmid_deviation_score', np.nan), + 'rprs_deviation_score': deviation_summary.get('rprs_deviation_score', np.nan), + 'deviation_from_expected_value': deviation_summary.get('deviation_from_expected_value', np.nan), + 'rprs_prior_assumed': ( + deviation_summary.get('rprs_prior_assumed', False) + or geometry_prior_assumed + ), + 'rprs_prior_assumed_note': ( + geometry_prior_assumed_note + if geometry_prior_assumed + else deviation_summary.get('rprs_prior_assumed_note') + ), + }) + + notes = [] + failure_reasons = [] + status = 'pass' + comparison_text = ( + f"Delta BIC={delta_bic:.2f}, Delta chi2={delta_chi2:.2f}" + if np.isfinite(delta_bic) and np.isfinite(delta_chi2) + else "model comparison unavailable" + ) + if geometry_prior_assumed_note: + notes.append(str(geometry_prior_assumed_note)) + + if not np.isfinite(delta_bic) or not np.isfinite(delta_chi2): + status = 'unknown' + notes.append("Transit-vs-flat model comparison was not finite.") + elif delta_chi2 <= 0: + status = 'fail' + notes.append("The flat/null model fits the lightcurve at least as well as the transit model.") + failure_reasons.append("the flat/null model fits the lightcurve at least as well as the transit model") + elif delta_bic < TRANSIT_QC_DELTA_BIC_FAIL_THRESHOLD: + status = 'fail' + notes.append( + "The transit model does not beat the flat/null model strongly enough to claim a detection." + ) + failure_reasons.append( + "the transit model does not beat the flat/null model strongly enough to claim a detection" + ) + elif delta_bic < TRANSIT_QC_DELTA_BIC_PASS_THRESHOLD: + status = 'marginal' + notes.append( + "The transit model is preferred over the flat/null model, but the evidence is only moderate." + ) + else: + notes.append("The transit model is strongly preferred over the flat/null model.") + + if np.isfinite(summary['rprs_sigma']): + notes.append( + f"Rp/R* fit precision diagnostic: {summary['rprs_sigma']:.2f}-sigma " + "(not used as a transit-detection veto)." + ) + + if np.isfinite(summary['duration_ratio']): + if summary.get('duration_consistency_note'): + notes.append(summary['duration_consistency_note']) + if ( + summary['duration_ratio'] < TRANSIT_QC_DURATION_RATIO_MIN + or summary['duration_ratio'] > TRANSIT_QC_DURATION_RATIO_MAX + ): + notes.append( + f"The measured transit duration is {summary['duration_ratio']:.2f}x the modeled duration." + ) + elif summary.get('duration_consistency_note'): + notes.append(summary['duration_consistency_note']) + + if np.isfinite(summary['eebls_depth_snr']) and summary['eebls_depth_snr'] < TRANSIT_QC_MIN_EEBLS_SNR: + notes.append( + f"EEBLS only found a weak box-like event (depth SNR={summary['eebls_depth_snr']:.2f})." + ) + + if use_deviation_from_expected_transit_in_qc: + notes.extend(deviation_summary.get('notes', [])) + if deviation_summary.get('failed'): + notes.append( + "The fit deviates far from the expected published Rp/R* value; " + "this now contributes through KTMF rather than acting as a hard QC veto." + ) + + ktmf_metric, ktmf_contributions = compute_transit_qc_ktmf(summary) + summary['ktmf_metric'] = ktmf_metric + summary['ktmf_contributions'] = ktmf_contributions + + if np.isfinite(ktmf_metric): + if ktmf_metric < TRANSIT_QC_KTMF_FAIL_THRESHOLD: + status = 'fail' + notes.append( + f"KTMF is {ktmf_metric:.2f}/5.00, below the fail threshold " + f"of {TRANSIT_QC_KTMF_FAIL_THRESHOLD:.2f}." + ) + failure_reasons.append( + f"KTMF is {ktmf_metric:.2f}/5.00, below the fail threshold " + f"of {TRANSIT_QC_KTMF_FAIL_THRESHOLD:.2f}" + ) + elif ktmf_metric < TRANSIT_QC_KTMF_PASS_THRESHOLD: + status = 'marginal' + notes.append( + f"KTMF is {ktmf_metric:.2f}/5.00, in the marginal range " + f"[{TRANSIT_QC_KTMF_FAIL_THRESHOLD:.2f}, {TRANSIT_QC_KTMF_PASS_THRESHOLD:.2f})." + ) + else: + status = 'pass' + notes.append( + f"KTMF is {ktmf_metric:.2f}/5.00, meeting the pass threshold " + f"of {TRANSIT_QC_KTMF_PASS_THRESHOLD:.2f}." + ) + + if status == 'pass': + summary_text = f"KTMF supports a pass-quality transit fit ({comparison_text})." + elif status == 'marginal': + summary_text = f"KTMF indicates a marginal transit fit ({comparison_text})." + elif status == 'fail': + if failure_reasons: + flat_model_only_failure = all( + "flat/null model" in reason or "does not beat the flat/null model" in reason + for reason in failure_reasons + ) + if flat_model_only_failure: + summary_text = ( + f"Transit detection not supported strongly enough against a flat/null model ({comparison_text})." + ) + else: + summary_text = ( + "Transit model is preferred over the flat/null model, but QC rejected the fit because " + + "; ".join(failure_reasons) + + f" ({comparison_text})." + ) + else: + summary_text = f"Transit detection QC rejected this fit ({comparison_text})." + else: + summary_text = f"Transit QC unavailable ({comparison_text})." + + summary.update({ + 'status': status, + 'summary': summary_text, + 'notes': notes, + }) + return summary + + +def annotate_transit_detection_qc(fit, summary=None): + if fit is None: + return + + summary = evaluate_transit_detection_qc(fit) if summary is None else dict(summary) + fit.transit_qc = summary + fit.transit_qc_computed = bool(summary.get('computed')) + fit.transit_qc_status = summary.get('status') + fit.transit_qc_summary = summary.get('summary') + fit.transit_qc_preferred_model = summary.get('preferred_model') + fit.transit_qc_delta_bic = summary.get('delta_bic') + fit.transit_qc_delta_chi2 = summary.get('delta_chi2') + fit.transit_qc_rprs_sigma = summary.get('rprs_sigma') + fit.transit_qc_duration_ratio = summary.get('duration_ratio') + fit.transit_qc_eebls_depth_snr = summary.get('eebls_depth_snr') + fit.transit_qc_residual_scatter = summary.get('residual_scatter') + fit.transit_qc_tmid_gaussianity_score = summary.get('tmid_gaussianity_score') + fit.transit_qc_tmid_gaussianity_score_uncertainty = summary.get( + 'tmid_gaussianity_score_uncertainty' + ) + fit.transit_qc_tmid_gaussianity_detail = summary.get('tmid_gaussianity_detail') + fit.transit_qc_deviation_from_expected_value = summary.get('deviation_from_expected_value') + fit.transit_qc_deviation_sigma_threshold = summary.get('deviation_sigma_threshold') + fit.transit_qc_expected_tmid_value = summary.get('expected_tmid') + fit.transit_qc_expected_tmid_unc = summary.get('expected_tmid_unc') + fit.transit_qc_expected_tmid_unc_minutes = summary.get('expected_tmid_unc_minutes') + fit.transit_qc_fitted_tmid = summary.get('fitted_tmid') + fit.transit_qc_fitted_rprs = summary.get('fitted_rprs') + fit.transit_qc_fitted_rprs_unc = summary.get('fitted_rprs_unc') + fit.transit_qc_tmid_deviation_days = summary.get('tmid_deviation_days') + fit.transit_qc_tmid_deviation_minutes = summary.get('tmid_deviation_minutes') + fit.transit_qc_tmid_deviation_threshold_minutes = summary.get('tmid_deviation_threshold_minutes') + fit.transit_qc_tmid_deviation_sigma = summary.get('tmid_deviation_sigma') + fit.transit_qc_rprs_deviation_fit_unc = summary.get('rprs_deviation_fit_unc') + fit.transit_qc_rprs_deviation_model_fit_unc = summary.get('rprs_deviation_model_fit_unc') + fit.transit_qc_rprs_deviation_data_fit_unc = summary.get('rprs_deviation_data_fit_unc') + fit.transit_qc_rprs_deviation_combined_fit_unc = summary.get('rprs_deviation_combined_fit_unc') + fit.transit_qc_rprs_deviation_expected_unc = summary.get('rprs_deviation_expected_unc') + fit.transit_qc_rprs_deviation_systematic_floor = summary.get('rprs_deviation_systematic_floor') + fit.transit_qc_rprs_deviation_unc = summary.get('rprs_deviation_unc') + fit.transit_qc_rprs_deviation_sigma = summary.get('rprs_deviation_sigma') + fit.transit_qc_expected_rprs_deviation_sigma = summary.get('rprs_deviation_sigma') + fit.transit_qc_ktmf_metric = summary.get('ktmf_metric') + fit.transit_qc_ktmf_contributions = summary.get('ktmf_contributions') + + +def lightcurve_fit_transit_qc_failure_reason(fit): + if fit is None: + return None + + transit_qc = getattr(fit, 'transit_qc', None) + if not isinstance(transit_qc, dict): + return None + + status = str(transit_qc.get('status', '')).strip().lower() + if status != 'fail': + return None + + summary = transit_qc.get('summary') + if isinstance(summary, str) and summary.strip(): + return summary.strip() + + return "Transit detection QC flagged this fit as a poor transit candidate." + + +def lightcurve_fit_transit_qc_passed(fit): + if fit is None: + return False + + transit_qc = getattr(fit, 'transit_qc', None) + if isinstance(transit_qc, dict): + return str(transit_qc.get('status', '')).strip().lower() == 'pass' + + return str(getattr(fit, 'transit_qc_status', '')).strip().lower() == 'pass' + + +def make_json_safe(value): + if isinstance(value, dict): + return {str(key): make_json_safe(subvalue) for key, subvalue in value.items()} + if isinstance(value, (list, tuple)): + return [make_json_safe(item) for item in value] + if isinstance(value, np.ndarray): + return [make_json_safe(item) for item in value.tolist()] + if isinstance(value, np.generic): + return value.item() + if isinstance(value, Path): + return str(value) + return value + + +def archive_exception_payload(action, exc): + traceback_text = ''.join( + traceback.format_exception(type(exc), exc, exc.__traceback__) + ).strip() + message = f"{action}: {type(exc).__name__}: {exc}" + try: + log_info(f"Warning: {message}", warn=True) + except Exception: + print(f"Warning: {message}", flush=True) + log.debug("%s\n%s", message, traceback_text) + return { + 'action': action, + 'error_type': type(exc).__name__, + 'message': str(exc), + 'traceback': traceback_text, + } + + +def failed_comparison_archive_dir(save_dir, comp_index): + base_dir = Path(save_dir) / "Diagnostics" + base_dir.mkdir(parents=True, exist_ok=True) + candidate_dir = base_dir / f"comp_{comp_index + 1}_failed" + if not candidate_dir.exists(): + return candidate_dir + + suffix = 2 + while True: + fallback_dir = base_dir / f"comp_{comp_index + 1}_failed_{suffix}" + if not fallback_dir.exists(): + return fallback_dir + suffix += 1 + + +def comparison_candidate_output_dir(save_dir, comp_index): + return Path(save_dir) / "Diagnostics" / f"comp{comp_index + 1}" + + +def triangle_plot_output_path(save_dir, planet_name, observation_date): + return ( + Path(save_dir) + / "Diagnostics" + / safe_output_filename("Triangle", planet_name, filename_date_token(observation_date), extension="png") + ) + + +def final_triangle_plot_output_path(save_dir, planet_name, observation_date): + return ( + Path(save_dir) + / "Diagnostics" + / safe_output_filename("FinalTriangle", planet_name, filename_date_token(observation_date), extension="png") + ) + + +def zoomed_final_triangle_plot_output_path(save_dir, planet_name, observation_date): + return ( + Path(save_dir) + / "Diagnostics" + / safe_output_filename("ZoomedTrianglePlot", planet_name, filename_date_token(observation_date), extension="png") + ) + + +def comparison_candidate_triangle_plot_output_path(save_dir, planet_name, observation_date, comp_index): + return ( + Path(save_dir) + / "working_artifacts" + / safe_output_filename( + f"Comp{int(comp_index) + 1}_Triangle", + planet_name, + filename_date_token(observation_date), + extension="png", + ) + ) + + +def comparison_candidate_label_from_output_dir(output_dir): + if output_dir is None: + return None + + for part in reversed(Path(output_dir).parts): + match = re.fullmatch(r"comp(\d+)", str(part), re.IGNORECASE) + if match: + return f"comparison candidate #{int(match.group(1))}" + return None + + +def _plot_triangle_for_output(fit, plot_title=None, required_keywords=(), **plot_kwargs): + plotter = getattr(fit, 'plot_triangle', None) + if not callable(plotter): + return None + + supported_kwargs = {} + for keyword in required_keywords: + if not callable_accepts_keyword(plotter, keyword): + return None + + if plot_title and callable_accepts_keyword(plotter, 'plot_title'): + supported_kwargs['plot_title'] = plot_title + for keyword, value in plot_kwargs.items(): + if callable_accepts_keyword(plotter, keyword): + supported_kwargs[keyword] = value + + return plotter(**supported_kwargs) + + +def _close_plot_figure(fig): + try: + plt.close(fig) + except TypeError: + pass + + +def save_final_triangle_plot(fit, save_dir, planet_name, observation_date, source_dir=None): + output_path = final_triangle_plot_output_path(save_dir, planet_name, observation_date) + zoomed_output_path = zoomed_final_triangle_plot_output_path(save_dir, planet_name, observation_date) + compatibility_path = triangle_plot_output_path(save_dir, planet_name, observation_date) + output_path.parent.mkdir(parents=True, exist_ok=True) + zoomed_output_path.parent.mkdir(parents=True, exist_ok=True) + compatibility_path.parent.mkdir(parents=True, exist_ok=True) + + source_label = comparison_candidate_label_from_output_dir(source_dir) + plot_title = "Final selected fit" + if source_label: + plot_title = f"{plot_title} ({source_label})" + fig = _plot_triangle_for_output(fit, plot_title=plot_title) + if fig is None: + return None + fig.savefig(output_path) + if compatibility_path != output_path: + try: + shutil.copy2(output_path, compatibility_path) + except Exception: + fig.savefig(compatibility_path) + _close_plot_figure(fig) + + zoomed_fig = None + try: + zoomed_title = f"{plot_title} (5-sigma zoom)" + zoomed_fig = _plot_triangle_for_output( + fit, + plot_title=zoomed_title, + required_keywords=('zoom_sigma',), + zoom_sigma=5.0, + ) + if zoomed_fig is not None: + zoomed_fig.savefig(zoomed_output_path) + except Exception as exc: + try: + log_info(f"Warning: Could not save zoomed final triangle plot: {exc}", warn=True) + except Exception: + pass + finally: + if zoomed_fig is not None: + _close_plot_figure(zoomed_fig) + return output_path + + +def estimate_transit_duration_samples_from_fit(fit, sample_count=1000, grid_size=1000): + if fit is None or not hasattr(fit, 'parameters') or not hasattr(fit, 'errors'): + return None, np.array([], dtype=float) + + fit_times = np.asarray(getattr(fit, 'time', []), dtype=float) + fit_times = fit_times[np.isfinite(fit_times)] + if fit_times.size < 2: + return None, np.array([], dtype=float) + + parameters = getattr(fit, 'parameters', {}) or {} + errors = getattr(fit, 'errors', {}) or {} + transit_times = np.linspace(np.nanmin(fit_times), np.nanmax(fit_times), int(grid_size)) + if transit_times.size < 2: + return None, np.array([], dtype=float) + + baseline_parameters = dict(parameters) + fit_transit_model = getattr(fit, '_transit_model', None) + if callable(fit_transit_model): + baseline_model = fit_transit_model(transit_times, baseline_parameters) + else: + baseline_model = transit(transit_times, baseline_parameters) + dt = float(np.nanmean(np.diff(transit_times))) + if not np.isfinite(dt) or dt <= 0: + return baseline_model, np.array([], dtype=float) + + duration_samples = [] + sample_count = max(1, int(sample_count)) + for _ in range(sample_count): + sampled_parameters = dict(parameters) + for key, error_value in errors.items(): + parameter_value = parameters.get(key) + if parameter_value is None: + continue + try: + numeric_error = float(error_value) + numeric_value = float(parameter_value) + except (TypeError, ValueError): + continue + if not np.isfinite(numeric_error) or numeric_error <= 0 or not np.isfinite(numeric_value): + continue + sampled_parameters[key] = np.random.normal(numeric_value, numeric_error) + + sampled_model = transit(transit_times, sampled_parameters) + in_transit_mask = np.asarray(sampled_model, dtype=float) < 1 + duration_samples.append(float(np.count_nonzero(in_transit_mask)) * dt) + + return baseline_model, np.asarray(duration_samples, dtype=float) + + +def build_comparison_candidate_adaptive_summary(comparison_calibration, psf_data, + use_adaptive_apertures=False, + adaptive_aperture_values=None, + adaptive_annulus_values=None, + fallback_sigma=np.nan): + if ( + not use_adaptive_apertures + or comparison_calibration is None + or comparison_calibration.get('method') == 'psf' + or adaptive_aperture_values is None + or adaptive_annulus_values is None + ): + return None + + aperture_index = comparison_calibration.get('a') + annulus_index = comparison_calibration.get('an') + if aperture_index is None or annulus_index is None: + return None + + aperture_grid = np.asarray(adaptive_aperture_values, dtype=float) + annulus_grid = np.asarray(adaptive_annulus_values, dtype=float) + if aperture_index >= aperture_grid.size or annulus_index >= annulus_grid.size: + return None + + return summarize_adaptive_aperture_usage( + psf_data['target'], + aperture_grid[aperture_index], + annulus_grid[annulus_index], + fallback_sigma=fallback_sigma, + ) + + +def build_comparison_candidate_transit_prior(p_dict, ld): + return { + 'rprs': p_dict['rprs'], + 'ars': p_dict['aRs'], + 'per': p_dict['pPer'], + 'inc': p_dict['inc'], + 'u0': ld[0], 'u1': ld[1], 'u2': ld[2], 'u3': ld[3], + 'ecc': p_dict['ecc'], + 'omega': p_dict['omega'], + 'tmid': p_dict['midT'], + 'a2': 0, + } + + +def build_search_restriction_prior_from_planet_dict(p_dict): + if not isinstance(p_dict, dict): + return {} + return { + 'pName': p_dict.get('pName'), + 'sName': p_dict.get('sName'), + 'rprs': p_dict.get('rprs'), + 'rprs_unc': p_dict.get('rprsUnc'), + 'ars': p_dict.get('aRs'), + 'ars_unc': p_dict.get('aRsUnc'), + 'inc': p_dict.get('inc'), + 'inc_unc': p_dict.get('incUnc'), + } + + +def estimate_rprs_data_uncertainty_from_lightcurve( + times, + flux_values, + flux_errors, + prior, + transit_depth_threshold_fraction=0.05, +): + payload = { + 'available': False, + 'data_rprs_uncertainty': np.nan, + 'rprs_data_uncertainty': np.nan, + 'note': 'Unavailable; the prior transit model could not be evaluated against the light curve.', + } + if not isinstance(prior, dict): + return payload + + try: + times = np.asarray(times, dtype=float).reshape(-1) + flux_values = np.asarray(flux_values, dtype=float).reshape(-1) + except (TypeError, ValueError): + return payload + if flux_errors is None: + flux_errors = np.full(flux_values.shape, np.nan, dtype=float) + else: + try: + flux_errors = np.asarray(flux_errors, dtype=float).reshape(-1) + except (TypeError, ValueError): + flux_errors = np.full(flux_values.shape, np.nan, dtype=float) + + if not (times.shape == flux_values.shape == flux_errors.shape): + return payload + if times.size < 3: + payload['note'] = 'Unavailable; too few light-curve points for a data-based Rp/R* uncertainty estimate.' + return payload + + try: + prior_transit = np.asarray(transit(times, prior), dtype=float).reshape(-1) + except Exception as exc: + payload['note'] = f"Unavailable; prior transit model evaluation failed ({describe_retry_exception(exc)})." + return payload + if prior_transit.shape != times.shape: + return payload + + finite_model = np.isfinite(prior_transit) + finite_flux = np.isfinite(flux_values) + if not np.any(finite_model & finite_flux): + return payload + + baseline_scale = solve_transit_qc_flux_baseline(prior_transit, flux_values, dataerr=flux_errors) + if not np.isfinite(baseline_scale) or baseline_scale <= 0: + baseline_scale = 1.0 + scaled_transit = prior_transit * baseline_scale + + class PrefitRprsUncertaintyFit: + pass + + dummy_fit = PrefitRprsUncertaintyFit() + dummy_fit.time = times + dummy_fit.data = flux_values + dummy_fit.dataerr = flux_errors + dummy_fit.transit = scaled_transit + dummy_fit.model = scaled_transit + dummy_fit.residuals = flux_values - scaled_transit + dummy_fit.airmass_model = np.ones_like(scaled_transit) + dummy_fit.parameters = dict(prior) + dummy_fit.errors = {'rprs': np.nan} + + empirical = fit_empirical_transit_uncertainty( + dummy_fit, + transit_depth_threshold_fraction=transit_depth_threshold_fraction, + ) + if not isinstance(empirical, dict) or not empirical.get('available'): + payload['note'] = ( + "Unavailable; the prior transit shape did not provide enough in-transit/out-of-transit " + "support for a data-based Rp/R* uncertainty estimate." + ) + return payload + + data_uncertainty = empirical.get('data_rprs_uncertainty') + try: + data_uncertainty = float(data_uncertainty) + except (TypeError, ValueError): + data_uncertainty = np.nan + if not np.isfinite(data_uncertainty) or data_uncertainty < 0: + return payload + + payload.update(empirical) + payload.update({ + 'available': True, + 'data_rprs_uncertainty': data_uncertainty, + 'rprs_data_uncertainty': data_uncertainty, + 'rprs_data_uncertainty_source': 'prefit_flux_residual_red_noise', + 'baseline_scale': float(baseline_scale), + 'note': ( + f"Estimated pre-fit Rp/R* data uncertainty {data_uncertainty:.6f} " + "from residual scatter around the prior transit shape." + ), + }) + return payload + + +def rprs_prior_window_half_widths(prior): + if not isinstance(prior, dict): + return None + try: + rprs = float(prior.get('rprs')) + percentage = float(RPRS_RANGE_RESTRICTION_PERCENTAGE) + except (TypeError, ValueError): + return None + if not np.isfinite(rprs) or rprs <= RPRS_SEARCH_BOUND_MIN: + return None + if not np.isfinite(percentage) or percentage < 0: + return None + + configured_half_width = abs(rprs) * percentage / 100.0 + data_uncertainty = prior.get('rprs_data_uncertainty', prior.get('data_rprs_uncertainty')) + try: + data_uncertainty = float(data_uncertainty) + except (TypeError, ValueError): + data_uncertainty = np.nan + try: + data_sigma = float(prior.get('rprs_data_uncertainty_bound_sigma', RPRS_DATA_UNCERTAINTY_BOUND_SIGMA)) + except (TypeError, ValueError): + data_sigma = RPRS_DATA_UNCERTAINTY_BOUND_SIGMA + if not np.isfinite(data_sigma) or data_sigma <= 0: + data_sigma = RPRS_DATA_UNCERTAINTY_BOUND_SIGMA + + data_half_width = np.nan + use_data_window = False + if np.isfinite(data_uncertainty) and data_uncertainty > configured_half_width: + data_half_width = float(data_sigma * data_uncertainty) + use_data_window = np.isfinite(data_half_width) and data_half_width > configured_half_width + + half_width = data_half_width if use_data_window else configured_half_width + if not np.isfinite(half_width) or half_width <= 0: + return None + return { + 'center': float(rprs), + 'configured_half_width': float(configured_half_width), + 'data_uncertainty': float(data_uncertainty) if np.isfinite(data_uncertainty) else np.nan, + 'data_sigma': float(data_sigma), + 'data_half_width': float(data_half_width) if np.isfinite(data_half_width) else np.nan, + 'half_width': float(half_width), + 'use_data_window': bool(use_data_window), + } + + +def rprs_prior_centered_bounds(prior): + widths = rprs_prior_window_half_widths(prior) + if not widths: + return None + center = widths['center'] + half_width = widths['half_width'] + lower_bound = max(float(RPRS_SEARCH_BOUND_MIN), center - half_width) + upper_bound = min(float(RPRS_SEARCH_BOUND_MAX), center + half_width) + if not np.isfinite(lower_bound) or not np.isfinite(upper_bound) or upper_bound <= lower_bound: + return None + return [float(lower_bound), float(upper_bound)] + + +def enrich_search_restriction_prior_with_rprs_data_uncertainty( + search_prior, + times, + flux_values, + flux_errors, + transit_prior, + context_label="light curve", +): + enriched = dict(search_prior) if isinstance(search_prior, dict) else {} + if not isinstance(transit_prior, dict): + return enriched + estimate = estimate_rprs_data_uncertainty_from_lightcurve( + times, + flux_values, + flux_errors, + transit_prior, + ) + if not estimate.get('available'): + return enriched + + data_uncertainty = estimate.get('data_rprs_uncertainty') + try: + data_uncertainty = float(data_uncertainty) + except (TypeError, ValueError): + data_uncertainty = np.nan + if not np.isfinite(data_uncertainty) or data_uncertainty < 0: + return enriched + + enriched['rprs_data_uncertainty'] = data_uncertainty + enriched['data_rprs_uncertainty'] = data_uncertainty + enriched['rprs_data_uncertainty_bound_sigma'] = RPRS_DATA_UNCERTAINTY_BOUND_SIGMA + enriched['rprs_data_uncertainty_payload'] = estimate + + widths = rprs_prior_window_half_widths(enriched) + if widths and widths.get('use_data_window'): + log_info( + f"Rp/R* data-derived uncertainty for the {context_label} is " + f"{data_uncertainty:.6f}, larger than the configured " + f"+/-{RPRS_RANGE_RESTRICTION_PERCENTAGE:.1f}% prior window " + f"({widths['configured_half_width']:.6f}); widening the prior-centered " + f"Rp/R* search half-width to {widths['data_sigma']:.1f} sigma " + f"({widths['half_width']:.6f})." + ) + return enriched + + +def widen_rprs_bounds_to_data_uncertainty_window(bounds, prior): + widened = clone_lightcurve_bounds(bounds) + if 'rprs' not in widened or not RPRS_RANGE_RESTRICTION_ENABLED: + return widened + widths = rprs_prior_window_half_widths(prior) + if not widths or not widths.get('use_data_window'): + return widened + window = rprs_prior_centered_bounds(prior) + if window is not None: + widened['rprs'] = window + return widened + + +def prepare_comparison_candidate_full_reduction_series(times, target_flux, comp_flux, airmass, + jd_times=None, adaptive_summary=None, + target_flux_error=None, comp_flux_error=None, + exposure_times_seconds=None, + gain_e_per_adu=None, + expected_transit_depth=None): + result = { + 'applied': False, + 'failure_reason': "the raw comparison-candidate photometry did not yield a usable light curve.", + 'filter_diagnostics': [], + 'debug_times': np.array([], dtype=float), + 'debug_target_flux': np.array([], dtype=float), + 'debug_comp_flux': np.array([], dtype=float), + 'debug_raw_ratio': np.array([], dtype=float), + 'debug_target_flux_error': np.array([], dtype=float), + 'debug_comp_flux_error': np.array([], dtype=float), + 'debug_relative_flux_error': np.array([], dtype=float), + 'initial_sigma_keep_mask': np.array([], dtype=bool), + 'prefit_raw_ratio_keep_mask': np.array([], dtype=bool), + 'time': np.array([], dtype=float), + 'flux': np.array([], dtype=float), + 'unc': np.array([], dtype=float), + 'airmass': np.array([], dtype=float), + 'jd_time': np.array([], dtype=float), + 'exposure_time_seconds': None, + 'target_flux': np.array([], dtype=float), + 'comp_flux': np.array([], dtype=float), + 'target_flux_error': np.array([], dtype=float), + 'comp_flux_error': np.array([], dtype=float), + 'source_indices': np.array([], dtype=int), + } + + prepared = prepare_lightcurve_fit_input_series( + times, + target_flux, + comp_flux, + airmass, + target_flux_error=target_flux_error, + comp_flux_error=comp_flux_error, + jd_times=jd_times, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=gain_e_per_adu, + expected_transit_depth=expected_transit_depth, + ) + result['filter_diagnostics'] = prepared.get('filter_diagnostics', []) + for key in ( + 'debug_times', + 'debug_target_flux', + 'debug_comp_flux', + 'debug_raw_ratio', + 'debug_target_flux_error', + 'debug_comp_flux_error', + 'debug_relative_flux_error', + 'initial_sigma_keep_mask', + 'prefit_raw_ratio_keep_mask', + ): + if key in prepared: + result[key] = prepared[key] + if not prepared.get('applied'): + result['failure_reason'] = prepared.get( + 'failure_reason', + "the raw comparison-candidate photometry did not yield a usable light curve.", + ) + return result + + good_times = np.asarray(prepared['time'], dtype=float) + good_flux = np.asarray(prepared['flux'], dtype=float) + good_unc = np.asarray(prepared['unc'], dtype=float) + good_airmass = np.asarray(prepared['airmass'], dtype=float) + good_jd_times = np.asarray(prepared['jd_time'], dtype=float) + good_exposure_times = prepared.get('exposure_time_seconds') + good_exposure_times = None if good_exposure_times is None else np.asarray(good_exposure_times, dtype=float) + good_target_flux = np.asarray(prepared['target_flux'], dtype=float) + good_comp_flux = np.asarray(prepared['comp_flux'], dtype=float) + good_target_flux_error = np.asarray(prepared.get('target_flux_error', []), dtype=float) + good_comp_flux_error = np.asarray(prepared.get('comp_flux_error', []), dtype=float) + if good_target_flux_error.shape != good_target_flux.shape: + good_target_flux_error = np.full(good_target_flux.shape, np.nan, dtype=float) + if good_comp_flux_error.shape != good_comp_flux.shape: + good_comp_flux_error = np.full(good_comp_flux.shape, np.nan, dtype=float) + source_indices = np.asarray(prepared['source_indices'], dtype=int) + + adaptive_clip_mask = np.zeros(good_times.shape[0], dtype=bool) + if adaptive_summary is not None: + aperture_series = np.asarray(adaptive_summary.get('aperture_series', []), dtype=float) + annulus_series = np.asarray(adaptive_summary.get('annulus_series', []), dtype=float) + if aperture_series.ndim == 1 and annulus_series.ndim == 1: + try: + selected_apertures = aperture_series[source_indices] + selected_annuli = annulus_series[source_indices] + except IndexError: + selected_apertures = None + selected_annuli = None + if ( + selected_apertures is not None + and selected_apertures.shape == good_times.shape + and selected_annuli.shape == good_times.shape + ): + adaptive_clip_mask = adaptive_aperture_outlier_mask( + selected_apertures, + selected_annuli, + ) + + if np.count_nonzero(~adaptive_clip_mask) < LIGHTCURVE_MIN_VALID_POINTS: + result['failure_reason'] = ( + "adaptive-aperture filtering left too few points for a stable comparison-candidate reduction." + ) + return result + + if np.any(adaptive_clip_mask): + good_times = good_times[~adaptive_clip_mask] + good_flux = good_flux[~adaptive_clip_mask] + good_unc = good_unc[~adaptive_clip_mask] + good_airmass = good_airmass[~adaptive_clip_mask] + good_jd_times = good_jd_times[~adaptive_clip_mask] + if good_exposure_times is not None: + good_exposure_times = good_exposure_times[~adaptive_clip_mask] + good_target_flux = good_target_flux[~adaptive_clip_mask] + good_comp_flux = good_comp_flux[~adaptive_clip_mask] + good_target_flux_error = good_target_flux_error[~adaptive_clip_mask] + good_comp_flux_error = good_comp_flux_error[~adaptive_clip_mask] + source_indices = source_indices[~adaptive_clip_mask] + + relative_flux_mask = relative_flux_filter_mask(good_flux) + if np.count_nonzero(relative_flux_mask) < LIGHTCURVE_MIN_VALID_POINTS: + result['failure_reason'] = ( + "the raw comparison-candidate light curve failed the relative-flux filter before full reduction." + ) + return result + + result.update({ + 'applied': True, + 'failure_reason': None, + 'time': good_times[relative_flux_mask], + 'flux': good_flux[relative_flux_mask], + 'unc': good_unc[relative_flux_mask], + 'airmass': good_airmass[relative_flux_mask], + 'jd_time': good_jd_times[relative_flux_mask], + 'exposure_time_seconds': ( + None if good_exposure_times is None else good_exposure_times[relative_flux_mask] + ), + 'target_flux': good_target_flux[relative_flux_mask], + 'comp_flux': good_comp_flux[relative_flux_mask], + 'target_flux_error': good_target_flux_error[relative_flux_mask], + 'comp_flux_error': good_comp_flux_error[relative_flux_mask], + 'source_indices': source_indices[relative_flux_mask], + }) + return result + + +def stellar_variability_scatter_from_flux(flux_values): + flux_values = np.asarray(flux_values, dtype=float) + valid = np.isfinite(flux_values) & (flux_values > 0) + if np.count_nonzero(valid) < LIGHTCURVE_MIN_VALID_POINTS: + return np.nan + values = flux_values[valid] + center = bn.nanmedian(values) + if not np.isfinite(center): + return np.nan + scatter = robust_scatter(values - center) + return float(scatter) if np.isfinite(scatter) and scatter >= 0 else np.nan + + +def prepare_stellar_variability_only_direct_series(times, target_flux, comp_flux, airmass, + jd_times=None, target_flux_error=None, + comp_flux_error=None, + exposure_times_seconds=None, + inherited_diagnostics=None): + result = { + 'applied': False, + 'failure_reason': ( + "the raw comparison-candidate photometry did not yield a usable " + "stellar-variability light curve." + ), + 'filter_diagnostics': list(inherited_diagnostics or []), + 'time': np.array([], dtype=float), + 'flux': np.array([], dtype=float), + 'unc': np.array([], dtype=float), + 'airmass': np.array([], dtype=float), + 'jd_time': np.array([], dtype=float), + 'exposure_time_seconds': None, + 'target_flux': np.array([], dtype=float), + 'comp_flux': np.array([], dtype=float), + 'target_flux_error': np.array([], dtype=float), + 'comp_flux_error': np.array([], dtype=float), + 'source_indices': np.array([], dtype=int), + } + + times = np.asarray(times, dtype=float).reshape(-1) + target_flux = np.asarray(target_flux, dtype=float).reshape(-1) + comp_flux = np.asarray(comp_flux, dtype=float).reshape(-1) + airmass = np.asarray(airmass, dtype=float).reshape(-1) + if jd_times is None: + jd_times = times + jd_times = np.asarray(jd_times, dtype=float).reshape(-1) + if not (times.shape == target_flux.shape == comp_flux.shape == airmass.shape == jd_times.shape): + result['failure_reason'] = "stellar-variability input arrays did not have matching lengths." + return result + + if target_flux_error is None: + target_flux_error = np.sqrt(np.clip(np.abs(target_flux), 1.0, None)) + target_flux_error = np.asarray(target_flux_error, dtype=float).reshape(-1) + if target_flux_error.shape != target_flux.shape: + target_flux_error = np.sqrt(np.clip(np.abs(target_flux), 1.0, None)) + if comp_flux_error is None: + comp_flux_error = np.sqrt(np.clip(np.abs(comp_flux), 1.0, None)) + comp_flux_error = np.asarray(comp_flux_error, dtype=float).reshape(-1) + if comp_flux_error.shape != comp_flux.shape: + comp_flux_error = np.sqrt(np.clip(np.abs(comp_flux), 1.0, None)) + + exposure_times = None + if exposure_times_seconds is not None: + exposure_times = np.asarray(exposure_times_seconds, dtype=float).reshape(-1) + if exposure_times.shape != times.shape: + exposure_times = None + + valid = ( + np.isfinite(times) + & np.isfinite(target_flux) + & np.isfinite(comp_flux) + & np.isfinite(airmass) + & (target_flux > 0) + & (comp_flux > 0) + ) + diagnostic = build_time_rejection_diagnostic( + "Stellar-variability direct finite/positive filter", + times, + valid, + note=( + "Dropped non-finite or non-positive target/reference photometry while building a " + "stellar-variability-only light curve without transit fitting." + ), + ) + if diagnostic is not None: + result['filter_diagnostics'].append(diagnostic) + if np.count_nonzero(valid) < LIGHTCURVE_MIN_VALID_POINTS: + result['failure_reason'] = ( + "too few valid points remained after finite/positive filtering for " + "stellar-variability-only analysis." + ) + return result + + source_indices = np.arange(times.shape[0], dtype=int) + times = times[valid] + target_flux = target_flux[valid] + comp_flux = comp_flux[valid] + airmass = airmass[valid] + jd_times = jd_times[valid] + target_flux_error = target_flux_error[valid] + comp_flux_error = comp_flux_error[valid] + source_indices = source_indices[valid] + if exposure_times is not None: + exposure_times = exposure_times[valid] + + raw_ratio = target_flux / comp_flux + relative_unc = np.abs(raw_ratio) * np.sqrt( + (target_flux_error / target_flux) ** 2 + + (comp_flux_error / comp_flux) ** 2 + ) + positive_unc = relative_unc[np.isfinite(relative_unc) & (relative_unc > 0)] + fallback_unc = float(np.nanmedian(positive_unc)) if positive_unc.size else 1.0e-6 + relative_unc = np.where(np.isfinite(relative_unc) & (relative_unc > 0), relative_unc, fallback_unc) + norm_flux, norm_unc, _ = normalize_flux_series_to_approximate_unity(raw_ratio, relative_unc) + + relative_flux_mask = relative_flux_filter_mask(norm_flux) & np.isfinite(norm_unc) & (norm_unc > 0) + diagnostic = build_time_rejection_diagnostic( + "Stellar-variability direct relative-flux filter", + times, + relative_flux_mask, + note="Dropped invalid normalized target/reference flux values before stellar-variability analysis.", + ) + if diagnostic is not None: + result['filter_diagnostics'].append(diagnostic) + if np.count_nonzero(relative_flux_mask) < LIGHTCURVE_MIN_VALID_POINTS: + result['failure_reason'] = ( + "too few valid normalized target/reference points remained for " + "stellar-variability-only analysis." + ) + return result + + result.update({ + 'applied': True, + 'failure_reason': None, + 'time': times[relative_flux_mask], + 'flux': norm_flux[relative_flux_mask], + 'unc': norm_unc[relative_flux_mask], + 'airmass': airmass[relative_flux_mask], + 'jd_time': jd_times[relative_flux_mask], + 'exposure_time_seconds': ( + None if exposure_times is None else exposure_times[relative_flux_mask] + ), + 'target_flux': target_flux[relative_flux_mask], + 'comp_flux': comp_flux[relative_flux_mask], + 'target_flux_error': target_flux_error[relative_flux_mask], + 'comp_flux_error': comp_flux_error[relative_flux_mask], + 'source_indices': source_indices[relative_flux_mask], + }) + return result + + +def build_stellar_variability_only_lightcurve( + prepared, + p_dict, + filter_diagnostics=None, + comp_index=None, + comp_label=None, + comp_position=None, + method_label=None, + plot_time_range=None, +): + if prepared is None or not prepared.get('applied'): + return None + + base_times = np.asarray(prepared.get('time'), dtype=float) + oot_mask, exclusion_summary = stellar_variability_out_of_transit_mask(base_times, p_dict) + filter_diagnostics = list(filter_diagnostics or []) + exclusion_diagnostic = build_time_rejection_diagnostic( + "Stellar-variability transit-window exclusion", + base_times, + oot_mask, + note=exclusion_summary.get('note'), + ) + if exclusion_diagnostic is not None: + filter_diagnostics.append(exclusion_diagnostic) + + if np.count_nonzero(oot_mask) < LIGHTCURVE_MIN_VALID_POINTS: + return None + + time = base_times[oot_mask] + data = np.asarray(prepared.get('flux'), dtype=float)[oot_mask] + dataerr = np.asarray(prepared.get('unc'), dtype=float)[oot_mask] + airmass = np.asarray(prepared.get('airmass'), dtype=float)[oot_mask] + jd_times = np.asarray(prepared.get('jd_time'), dtype=float)[oot_mask] + source_indices = np.asarray(prepared.get('source_indices'), dtype=int)[oot_mask] + exposure_times_seconds = prepared.get('exposure_time_seconds') + if exposure_times_seconds is not None: + exposure_times_seconds = np.asarray(exposure_times_seconds, dtype=float)[oot_mask] + exposure_times_days = exposure_times_seconds / 86400.0 + else: + exposure_times_days = None + + finite = ( + np.isfinite(time) + & np.isfinite(data) + & np.isfinite(dataerr) + & (data > 0) + & (dataerr > 0) + & np.isfinite(airmass) + ) + if np.count_nonzero(finite) < LIGHTCURVE_MIN_VALID_POINTS: + return None + + time = time[finite] + data = data[finite] + dataerr = dataerr[finite] + airmass = airmass[finite] + jd_times = jd_times[finite] + source_indices = source_indices[finite] + if exposure_times_days is not None: + exposure_times_days = exposure_times_days[finite] + exposure_times_seconds = exposure_times_seconds[finite] + + phase = get_phase(time, p_dict.get('pPer', 1.0), p_dict.get('midT', np.nan)) + if np.any(np.isfinite(time)): + time_upsample = np.linspace(np.nanmin(time), np.nanmax(time), 1000) + else: + time_upsample = np.array([], dtype=float) + phase_upsample = get_phase(time_upsample, p_dict.get('pPer', 1.0), p_dict.get('midT', np.nan)) + flat_model = np.ones(time.shape, dtype=float) + flat_upsample = np.ones(time_upsample.shape, dtype=float) + scatter = stellar_variability_scatter_from_flux(data) + residuals = data - 1.0 + + parameters = { + 'tmid': p_dict.get('midT', np.nan), + 'per': p_dict.get('pPer', np.nan), + 'rprs': p_dict.get('rprs', np.nan), + 'ars': p_dict.get('aRs', np.nan), + 'inc': p_dict.get('inc', np.nan), + 'ecc': p_dict.get('ecc', 0.0), + 'omega': p_dict.get('omega', 0.0), + 'a0': 1.0, + 'a1': 1.0, + 'a2': 0.0, + } + errors = { + 'tmid': p_dict.get('midTUnc', np.nan), + 'per': p_dict.get('pPerUnc', np.nan), + 'rprs': p_dict.get('rprsUnc', np.nan), + 'ars': p_dict.get('aRsUnc', np.nan), + 'inc': p_dict.get('incUnc', np.nan), + 'a0': 0.0, + 'a1': 0.0, + 'a2': 0.0, + } + + fit = SimpleNamespace( + stellar_variability_only=True, + time=time, + jd_times=jd_times, + data=data, + dataerr=dataerr, + airmass=airmass, + airmass_model=np.ones(time.shape, dtype=float), + wf=np.ones(time.shape, dtype=float), + transit=flat_model, + model=flat_model, + detrended=data, + detrendederr=dataerr, + residuals=residuals, + phase=phase, + time_upsample=time_upsample, + phase_upsample=phase_upsample, + transit_upsample=flat_upsample, + exposure_times_days=exposure_times_days, + parameters=parameters, + errors=errors, + bounds={}, + sample_parameters={}, + sample_errors={}, + ns_type=None, + chi2=float(np.nansum((residuals / dataerr) ** 2)) if dataerr.size else np.nan, + frame_filter_diagnostics=filter_diagnostics, + stellar_variability_reference_comp_index=comp_index, + stellar_variability_reference_label=comp_label, + stellar_variability_reference_position=comp_position, + stellar_variability_method_label=method_label, + stellar_variability_scatter=scatter, + stellar_variability_transit_exclusion=exclusion_summary, + stellar_variability_source_indices=source_indices, + stellar_variability_target_flux=np.asarray(prepared.get('target_flux'), dtype=float)[oot_mask][finite], + stellar_variability_comp_flux=np.asarray(prepared.get('comp_flux'), dtype=float)[oot_mask][finite], + stellar_variability_target_flux_error=np.asarray(prepared.get('target_flux_error'), dtype=float)[oot_mask][finite], + stellar_variability_comp_flux_error=np.asarray(prepared.get('comp_flux_error'), dtype=float)[oot_mask][finite], + stellar_variability_exposure_times_seconds=exposure_times_seconds, + airmass_fit_skipped=True, + airmass_correction_note=( + "Intentionally not applied to stellar variability; the raw target/reference ratio is " + "preserved so real time-dependent variability is not fitted away." + ), + transit_qc={ + 'status': 'SKIPPED', + 'summary': 'Stellar variability only mode skipped transit fitting.', + 'residual_scatter': scatter, + }, + ) + fit = apply_plot_time_range(fit, time if plot_time_range is None else plot_time_range) + return fit + + +def build_stellar_variability_only_lightcurve_from_fluxes( + times, + target_flux, + comp_flux, + airmass, + p_dict, + jd_times=None, + adaptive_summary=None, + target_flux_error=None, + comp_flux_error=None, + exposure_times_seconds=None, + gain_e_per_adu=None, + filter_diagnostics=None, + comp_index=None, + comp_label=None, + comp_position=None, + method_label=None, + plot_time_range=None, +): + prepared = prepare_comparison_candidate_full_reduction_series( + times, + target_flux, + comp_flux, + airmass, + jd_times=jd_times, + adaptive_summary=adaptive_summary, + target_flux_error=target_flux_error, + comp_flux_error=comp_flux_error, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=gain_e_per_adu, + expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), + ) + if not prepared.get('applied'): + prepared = prepare_stellar_variability_only_direct_series( + times, + target_flux, + comp_flux, + airmass, + jd_times=jd_times, + target_flux_error=target_flux_error, + comp_flux_error=comp_flux_error, + exposure_times_seconds=exposure_times_seconds, + inherited_diagnostics=prepared.get('filter_diagnostics', []), + ) + diagnostics = list(filter_diagnostics or []) + diagnostics.extend(prepared.get('filter_diagnostics', [])) + fit = build_stellar_variability_only_lightcurve( + prepared, + p_dict, + filter_diagnostics=diagnostics, + comp_index=comp_index, + comp_label=comp_label, + comp_position=comp_position, + method_label=method_label, + plot_time_range=plot_time_range, + ) + return fit, prepared + + +def comparison_candidate_coverage_priority(assessment): + if not isinstance(assessment, dict) or not assessment.get('valid'): + return 4 + + pre_points = int(assessment.get('pre_ingress_points', 0) or 0) + post_points = int(assessment.get('post_egress_points', 0) or 0) + transit_fraction = coerce_finite_transit_qc_scalar( + assessment.get('transit_fraction_observed', np.nan) + ) + has_two_sided_oot = pre_points > 0 and post_points > 0 + covers_full_window = ( + bool(assessment.get('covers_ingress', False)) + and bool(assessment.get('covers_mid_transit', False)) + and bool(assessment.get('covers_egress', False)) + ) + + if has_two_sided_oot and covers_full_window: + return 0 + if has_two_sided_oot: + return 1 + if np.isfinite(transit_fraction) and transit_fraction >= 0.75 and assessment.get('covers_mid_transit', False): + return 2 + if np.isfinite(transit_fraction) and transit_fraction > 0: + return 3 + return 4 + + +def is_low_one_sided_expected_transit_coverage(assessment): + if not isinstance(assessment, dict) or not assessment.get('valid'): + return False + + pre_points = int(assessment.get('pre_ingress_points', 0) or 0) + post_points = int(assessment.get('post_egress_points', 0) or 0) + success_label = str(assessment.get('success_label', '')).strip().lower() + expected_successful = bool(assessment.get('expected_successful', False)) + return (pre_points == 0 or post_points == 0) and ( + success_label in ('very low', 'low') or not expected_successful + ) + + +def partial_transit_geometry_prior_assumption_mode(assessment): + if not isinstance(assessment, dict) or not assessment.get('valid'): + return None + + transit_fraction = coerce_finite_transit_qc_scalar( + assessment.get('transit_fraction_observed', np.nan) + ) + in_transit_points = int(assessment.get('in_transit_points', 0) or 0) + if ( + (not np.isfinite(transit_fraction) or transit_fraction <= 0) + and in_transit_points <= 0 + ): + return None + + pre_points = int(assessment.get('pre_ingress_points', 0) or 0) + post_points = int(assessment.get('post_egress_points', 0) or 0) + if pre_points == 0 and post_points == 0: + return 'tmid_baseline_airmass' + if pre_points == 0 or post_points == 0: + return 'tmid_only' + return None + + +def partial_transit_geometry_prior_assumption_fixed_error(key, prior, search_restriction_prior): + aliases = { + 'rprs': ('rprs_unc', 'rprsUnc', 'rprs_error', 'rprsErr', 'rprs_data_uncertainty'), + 'ars': ('ars_unc', 'aRsUnc', 'ars_error', 'aRsErr'), + 'inc': ('inc_unc', 'incUnc', 'inc_error', 'incErr'), + 'b': ('b_unc', 'impact_parameter_unc', 'impactParameterUnc'), + } + for source in (search_restriction_prior, prior): + if not isinstance(source, dict): + continue + for alias in aliases.get(key, ()): + value = coerce_finite_transit_qc_scalar(source.get(alias, np.nan)) + if np.isfinite(value) and value >= 0: + return float(value) + return 0.0 + + +def estimate_fixed_airmass_coefficient_error(flux_values, flux_errors, airmass): + flux_values = np.asarray([] if flux_values is None else flux_values, dtype=float) + flux_errors = np.asarray([] if flux_errors is None else flux_errors, dtype=float) + airmass = np.asarray([] if airmass is None else airmass, dtype=float) + if not (flux_values.shape == flux_errors.shape == airmass.shape): + return None + + finite = ( + np.isfinite(flux_values) + & (flux_values > 0) + & np.isfinite(flux_errors) + & (flux_errors > 0) + & np.isfinite(airmass) + ) + if int(np.count_nonzero(finite)) < 2: + return None + + airmass_span_value = np.nanmax(airmass[finite]) - np.nanmin(airmass[finite]) + if not np.isfinite(airmass_span_value) or airmass_span_value <= 0: + return None + + relative_errors = flux_errors[finite] / np.maximum(flux_values[finite], np.finfo(float).eps) + relative_error = float(np.nanmedian(relative_errors)) + if not np.isfinite(relative_error) or relative_error < 0: + return None + + return float(relative_error / airmass_span_value) + + +def partial_transit_geometry_prior_assumption_note(mode, assessment, sampled_parameters): + observed_segment = assessment.get('observed_segment') or 'partial transit' + pre_points = int(assessment.get('pre_ingress_points', 0) or 0) + post_points = int(assessment.get('post_egress_points', 0) or 0) + sampled_text = ", ".join(sampled_parameters) if sampled_parameters else "no free parameters" + if mode == 'tmid_baseline_airmass': + return ( + "Applied prior-assumed transit geometry for a no-out-of-transit partial light curve; " + "Rp/R*, a/Rs, and inclination/impact parameter were fixed to the input priors because " + f"the observation contains {pre_points} pre-ingress and {post_points} post-egress " + f"out-of-transit point(s) ({observed_segment}). The nested fit keeps baseline/airmass " + f"terms simultaneous with Tmid; sampled parameter(s): {sampled_text}." + ) + return ( + "Applied prior-assumed transit geometry for a one-sided partial light curve; Rp/R*, a/Rs, " + "and inclination/impact parameter were fixed to the input priors because the transit shape " + f"is baseline-degenerate with {pre_points} pre-ingress and {post_points} post-egress " + f"out-of-transit point(s) ({observed_segment}). Sampled parameter(s): {sampled_text}." + ) + + +def ensure_simultaneous_baseline_airmass_bounds_for_no_oot(prior, bounds, flux_values, airmass): + fit_a2 = not should_skip_airmass_fit(airmass) + ensure_pre_final_ultranest_baseline_bounds( + prior, + bounds, + flux_values, + fit_a2=fit_a2, + ) + + +def apply_partial_transit_geometry_prior_assumption( + prior, + bounds, + assessment, + flux_values=None, + flux_errors=None, + airmass=None, + fixed_parameter_errors=None, + search_restriction_prior=None, +): + mode = partial_transit_geometry_prior_assumption_mode(assessment) + local_prior = dict(prior) if isinstance(prior, dict) else {} + local_bounds = clone_lightcurve_bounds(bounds) + local_fixed_errors = ( + dict(fixed_parameter_errors) + if isinstance(fixed_parameter_errors, dict) + else {} + ) + payload = { + 'applied': False, + 'mode': None, + 'note': None, + 'fixed_parameters': [], + 'sampled_parameters': list(local_bounds.keys()), + } + if mode is None: + return local_prior, local_bounds, local_fixed_errors, payload + + fixed_parameters = [] + for key in ('rprs', 'ars', 'inc', 'b'): + if key in local_bounds: + local_bounds.pop(key, None) + fixed_parameters.append(key) + elif key in local_prior: + fixed_parameters.append(key) + if key in local_prior and key not in local_fixed_errors: + local_fixed_errors[key] = partial_transit_geometry_prior_assumption_fixed_error( + key, + local_prior, + search_restriction_prior, + ) + + if mode == 'tmid_baseline_airmass': + ensure_simultaneous_baseline_airmass_bounds_for_no_oot( + local_prior, + local_bounds, + flux_values, + airmass, + ) + else: + for key in ('a0', 'a1', 'a2'): + local_bounds.pop(key, None) + if 'a2' in local_prior and 'a2' not in local_fixed_errors: + a2_error = estimate_fixed_airmass_coefficient_error( + flux_values, + flux_errors, + airmass, + ) + if a2_error is not None: + local_fixed_errors['a2'] = a2_error + + sampled_parameters = list(local_bounds.keys()) + payload.update({ + 'applied': True, + 'mode': mode, + 'fixed_parameters': sorted(set(fixed_parameters)), + 'sampled_parameters': sampled_parameters, + }) + payload['note'] = partial_transit_geometry_prior_assumption_note( + mode, + assessment, + sampled_parameters, + ) + return local_prior, local_bounds, local_fixed_errors, payload + + +def partial_transit_geometry_retry_limits(assessment): + active = is_low_one_sided_expected_transit_coverage(assessment) + note = None + if active: + note = ( + "Skipped; pre-UltraNest coverage is one-sided/LOW, so EXOTIC does not expand this " + "geometry posterior range while the transit shape is baseline-degenerate." + ) + return { + 'active': active, + 'note': note, + 'max_retries': { + 'rprs': PARTIAL_COVERAGE_RPRS_POSTERIOR_MAX_RETRIES, + 'ars': PARTIAL_COVERAGE_ARS_POSTERIOR_MAX_RETRIES, + 'b': PARTIAL_COVERAGE_IMPACT_PARAMETER_POSTERIOR_MAX_RETRIES, + }, + } + + +def score_comparison_candidate_lightcurve_scout(prepared_series, eebls_summary, prior): + result = { + 'score': np.nan, + 'scatter': np.nan, + 'scatter_score': np.nan, + 'depth_score': np.nan, + 'eebls_score': np.nan, + 'eebls_snr': np.nan, + 'eebls_depth': np.nan, + 'expected_depth': np.nan, + } + if not isinstance(prepared_series, dict) or not prepared_series.get('applied'): + return result + + flux = np.asarray(prepared_series.get('flux', []), dtype=float) + finite_flux = flux[np.isfinite(flux) & (flux > 0)] + if finite_flux.size < LIGHTCURVE_MIN_VALID_POINTS: + return result + + baseline = bn.nanmedian(finite_flux) + if not np.isfinite(baseline) or baseline <= 0: + return result + + normalized_flux = finite_flux / baseline + scatter = robust_scatter(normalized_flux - bn.nanmedian(normalized_flux)) + rprs = coerce_finite_transit_qc_scalar(prior.get('rprs', np.nan) if isinstance(prior, dict) else np.nan) + expected_depth = rprs ** 2 if np.isfinite(rprs) and rprs >= 0 else np.nan + if np.isfinite(scatter): + result['scatter'] = float(scatter) + if np.isfinite(expected_depth): + result['expected_depth'] = float(expected_depth) + + depth = coerce_finite_transit_qc_scalar((eebls_summary or {}).get('depth', np.nan)) + depth_snr = coerce_finite_transit_qc_scalar((eebls_summary or {}).get('depth_snr', np.nan)) + if np.isfinite(depth): + result['eebls_depth'] = float(depth) + if np.isfinite(depth_snr): + result['eebls_snr'] = float(depth_snr) + + scatter_reference = expected_depth if np.isfinite(expected_depth) and expected_depth > 0 else 0.005 + if np.isfinite(scatter) and scatter >= 0: + result['scatter_score'] = float(np.clip(1.0 / (1.0 + scatter / max(scatter_reference, 1e-6)), 0.0, 1.0)) + + if np.isfinite(depth) and depth > 0 and np.isfinite(expected_depth) and expected_depth > 0: + depth_ratio = depth / expected_depth + if np.isfinite(depth_ratio) and depth_ratio > 0: + result['depth_score'] = float(np.clip(np.exp(-abs(np.log(depth_ratio)) / np.log(2.0)), 0.0, 1.0)) + + if np.isfinite(depth_snr) and depth_snr > 0: + result['eebls_score'] = float(np.clip(depth_snr / 8.0, 0.0, 1.0)) + + components = [ + (0.45, result['scatter_score']), + (0.35, result['depth_score']), + (0.20, result['eebls_score']), + ] + available = [(weight, value) for weight, value in components if np.isfinite(value)] + if available: + weight_sum = sum(weight for weight, _ in available) + result['score'] = float(sum(weight * value for weight, value in available) / weight_sum) + return result + + +def build_comparison_candidate_preflight(times, jd_times, airmass, ld, p_dict, target_flux, comp_flux, + target_flux_error=None, comp_flux_error=None, + exposure_times_seconds=None, + gain_e_per_adu=None, + adaptive_summary=None, use_eebls_to_initialize_tmid_and_bounds=True): + prepared = prepare_comparison_candidate_full_reduction_series( + times, + target_flux, + comp_flux, + airmass, + jd_times=jd_times, + adaptive_summary=adaptive_summary, + target_flux_error=target_flux_error, + comp_flux_error=comp_flux_error, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=gain_e_per_adu, + expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), + ) + try: + prior = build_comparison_candidate_transit_prior(p_dict, ld) + except (KeyError, IndexError, TypeError, ValueError): + return { + 'prepared_series': prepared, + 'coverage_assessment': None, + 'coverage_priority': 4, + 'eebls_summary': None, + 'tmid_search_summary': None, + 'duration_prior': None, + 'scout': {'score': np.nan}, + } + + duration_prior = build_single_transit_duration_prior(p_dict) + eebls_summary = None + tmid_search_summary = None + coverage_assessment = None + scout = {'score': np.nan} + + if prepared.get('applied'): + good_times = np.asarray(prepared['time'], dtype=float) + good_flux = np.asarray(prepared['flux'], dtype=float) + good_unc = np.asarray(prepared['unc'], dtype=float) + expected_duration = estimate_transit_duration_from_prior_geometry(prior) + tmid_search_summary = estimate_ephemeris_tmid_and_bounds( + good_times, + p_dict.get('midT', prior.get('tmid', np.nan)), + prior['per'], + p_dict.get('midTUnc', 0.01), + p_dict.get('pPerUnc', 0.0), + expected_duration=expected_duration, + sigma_multiplier=35.0, + ) + prior['tmid'] = tmid_search_summary['tmid'] + lower, upper = tmid_search_summary['bounds'] + if use_eebls_to_initialize_tmid_and_bounds: + eebls_summary = estimate_tmid_and_bounds_with_eebls( + good_times, + good_flux, + good_unc, + prior, + [lower, upper], + ) + else: + eebls_summary = {'applied': False, 'depth': np.nan, 'depth_snr': np.nan} + coverage_assessment = build_expected_transit_coverage_assessment( + good_times, + prior, + flux_values=good_flux, + flux_errors=good_unc, + tmid_search_summary=tmid_search_summary, + duration_prior=duration_prior, + ) + scout = score_comparison_candidate_lightcurve_scout(prepared, eebls_summary, prior) + + return { + 'prepared_series': prepared, + 'coverage_assessment': coverage_assessment, + 'coverage_priority': comparison_candidate_coverage_priority(coverage_assessment), + 'eebls_summary': eebls_summary, + 'tmid_search_summary': tmid_search_summary, + 'duration_prior': duration_prior, + 'scout': scout, + } + + +def comparison_preflight_field_band_limit(plans): + finite_scores = [ + plan['summary'].get('aggregate_score', np.nan) + for plan in plans + if np.isfinite(plan['summary'].get('aggregate_score', np.nan)) + ] + if not finite_scores: + return np.inf + best_score = float(min(finite_scores)) + return best_score + max( + COMPARISON_PREFLIGHT_FIELD_SCORE_ABSOLUTE_BAND, + abs(best_score) * COMPARISON_PREFLIGHT_FIELD_SCORE_RELATIVE_BAND, + ) + + +def rank_comparison_candidate_preflight_plans(plans): + if not plans: + return [] + + field_band_limit = comparison_preflight_field_band_limit(plans) + + def sort_key(plan): + preflight = plan.get('preflight') or {} + scout = preflight.get('scout') or {} + aggregate_score = plan['summary'].get('aggregate_score', np.inf) + finite_aggregate = aggregate_score if np.isfinite(aggregate_score) else np.inf + close_field_band = 0 if finite_aggregate <= field_band_limit else 1 + scout_score = scout.get('score', np.nan) + scout_sort = -float(scout_score) if np.isfinite(scout_score) else np.inf + return ( + int(preflight.get('coverage_priority', 4)), + close_field_band, + scout_sort, + finite_aggregate, + plan.get('field_rank', np.inf), + ) + + return sorted(plans, key=sort_key) + + +def log_comparison_candidate_preflight_order(plans, ranked_plans): + if not plans or not ranked_plans: + return + original_order = [plan['summary'].get('comp_index') for plan in plans] + ranked_order = [plan['summary'].get('comp_index') for plan in ranked_plans] + if original_order == ranked_order: + return + + log_info( + "Comparison-star target-fit order adjusted by pre-UltraNest coverage/scout preflight " + "(Tmid remains free; scout uses coverage, scatter, depth plausibility, and EEBLS SNR)." + ) + for new_rank, plan in enumerate(ranked_plans, start=1): + summary = plan['summary'] + preflight = plan.get('preflight') or {} + coverage = preflight.get('coverage_assessment') or {} + scout = preflight.get('scout') or {} + scout_score = scout.get('score', np.nan) + scout_text = "n/a" if not np.isfinite(scout_score) else f"{scout_score:.3f}" + scatter = scout.get('scatter', np.nan) + scatter_text = "n/a" if not np.isfinite(scatter) else f"{100.0 * scatter:.4f}%" + eebls_snr = scout.get('eebls_snr', np.nan) + eebls_text = "n/a" if not np.isfinite(eebls_snr) else f"{eebls_snr:.2f}" + label = summary.get('label', f"Comp {summary.get('comp_index', 0) + 1}") + log_info( + f" Preflight rank {new_rank}: {label} " + f"(field rank {plan.get('field_rank', 0) + 1}), coverage_priority={preflight.get('coverage_priority', 'n/a')}, " + f"pre/post={coverage.get('pre_ingress_points', 'n/a')}/{coverage.get('post_egress_points', 'n/a')}, " + f"scout={scout_text}, scatter={scatter_text}, eebls_snr={eebls_text}." + ) + + +def match_time_subset_indices(full_times, subset_times, rtol=1e-10, atol=1e-10): + full_times = np.asarray(full_times, dtype=float).reshape(-1) + subset_times = np.asarray(subset_times, dtype=float).reshape(-1) + if subset_times.size == 0: + return np.array([], dtype=int) + if full_times.size < subset_times.size: + return None + + matched_indices = [] + search_start = 0 + for subset_time in subset_times: + if not np.isfinite(subset_time): + return None + remaining = full_times[search_start:] + matches = np.flatnonzero(np.isclose(remaining, subset_time, rtol=rtol, atol=atol)) + if matches.size == 0: + return None + matched_index = search_start + int(matches[0]) + matched_indices.append(matched_index) + search_start = matched_index + 1 + + return np.asarray(matched_indices, dtype=int) + + +def finalize_comparison_candidate_full_reduction(times, target_flux, comp_flux, airmass, ld, p_dict, + jd_times=None, + target_flux_error=None, + comp_flux_error=None, + exposure_times_seconds=None, + gain_e_per_adu=None, + disable_vertical_flux_normalization=False, + detrend_on_outoftransit_baseline=True, + use_impactparameter_rather_than_inclination_to_fit=True, + use_eebls_to_initialize_tmid_and_bounds=True, + plot_time_range=None, + baseline_duration_multiplier=FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT, + adaptive_summary=None, + run_fast_ultranest_before_final_run=FAST_ULTRANEST_BEFORE_FINAL_RUN_DEFAULT, + run_final_fit_phase_residual_clip=FINAL_FIT_PHASE_RESIDUAL_CLIP_DEFAULT, + run_final_residual_rejection=FINAL_RESIDUAL_REJECTION_DEFAULT, + precomputed_candidate_series=None): + result = { + 'applied': False, + 'fit': None, + 'good_times': np.array([], dtype=float), + 'good_flux': np.array([], dtype=float), + 'good_unc': np.array([], dtype=float), + 'good_airmass': np.array([], dtype=float), + 'good_jd_times': np.array([], dtype=float), + 'good_exposure_times_seconds': None, + 'good_target_flux': np.array([], dtype=float), + 'good_comp_flux': np.array([], dtype=float), + 'good_target_flux_error': np.array([], dtype=float), + 'good_comp_flux_error': np.array([], dtype=float), + 'source_indices': np.array([], dtype=int), + 'data_highres': None, + 'duration_samples': np.array([], dtype=float), + 'failure_reason': "full candidate reduction did not run.", + 'filter_diagnostics': [], + 'note': None, + } + if precomputed_candidate_series is None: + prepared = prepare_comparison_candidate_full_reduction_series( + times, + target_flux, + comp_flux, + airmass, + jd_times=jd_times, + adaptive_summary=adaptive_summary, + target_flux_error=target_flux_error, + comp_flux_error=comp_flux_error, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=gain_e_per_adu, + expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), + ) + else: + prepared = precomputed_candidate_series + result['filter_diagnostics'] = prepared.get('filter_diagnostics', []) + if not prepared.get('applied'): + result['failure_reason'] = prepared.get( + 'failure_reason', + "the raw comparison-candidate photometry did not yield a usable light curve.", + ) + return result + + good_times = np.asarray(prepared['time'], dtype=float) + good_flux = np.asarray(prepared['flux'], dtype=float) + good_unc = np.asarray(prepared['unc'], dtype=float) + good_airmass = np.asarray(prepared['airmass'], dtype=float) + good_jd_times = np.asarray(prepared['jd_time'], dtype=float) + good_exposure_times = prepared.get('exposure_time_seconds') + good_exposure_times = None if good_exposure_times is None else np.asarray(good_exposure_times, dtype=float) + good_target_flux = np.asarray(prepared['target_flux'], dtype=float) + good_comp_flux = np.asarray(prepared['comp_flux'], dtype=float) + good_target_flux_error = np.asarray(prepared.get('target_flux_error', []), dtype=float) + good_comp_flux_error = np.asarray(prepared.get('comp_flux_error', []), dtype=float) + if good_target_flux_error.shape != good_target_flux.shape: + good_target_flux_error = np.full(good_target_flux.shape, np.nan, dtype=float) + if good_comp_flux_error.shape != good_comp_flux.shape: + good_comp_flux_error = np.full(good_comp_flux.shape, np.nan, dtype=float) + source_indices = np.asarray(prepared['source_indices'], dtype=int) + + prior = build_comparison_candidate_transit_prior(p_dict, ld) + + expected_duration = estimate_transit_duration_from_prior_geometry(prior) + tmid_search_summary = estimate_ephemeris_tmid_and_bounds( + good_times, + p_dict['midT'], + prior['per'], + p_dict['midTUnc'], + p_dict['pPerUnc'], + expected_duration=expected_duration, + sigma_multiplier=35.0, + ) + prior['tmid'] = tmid_search_summary['tmid'] + lower, upper = tmid_search_summary['bounds'] + + eebls_search_summary = None + if use_eebls_to_initialize_tmid_and_bounds: + eebls_search_summary = estimate_tmid_and_bounds_with_eebls( + good_times, + good_flux, + good_unc, + prior, + [lower, upper], + ) + if eebls_search_summary.get('applied'): + prior['tmid'] = eebls_search_summary['tmid'] + lower, upper = eebls_search_summary['bounds'] + + skip_final_airmass_fit = False + airmass_skip_note = None + final_airmass_span = airmass_span(good_airmass) + if should_skip_airmass_fit(good_airmass): + skip_final_airmass_fit = True + airmass_skip_note = ( + f"Skipped (airmass span {final_airmass_span:.4f} <= {AIRMASS_FLAT_RANGE_THRESHOLD:.2f}); " + "no airmass correction applied." + ) + + search_restriction_prior = build_search_restriction_prior_from_planet_dict(p_dict) + bounds = build_initial_transit_bounds( + prior, + [lower, upper], + ars_unc=p_dict.get('aRsUnc'), + search_restriction_prior=search_restriction_prior, + ) + apply_vertical_flux_normalization_bound( + prior, + bounds, + good_flux, + disable_vertical_flux_normalization, + ) + if not skip_final_airmass_fit: + bounds['a2'] = [-3, 3] + ensure_pre_final_ultranest_baseline_bounds(prior, bounds, good_flux, fit_a2=True) + + debug_phase_clip_keep_mask = None + prefit_kwargs = { + 'jd_times': good_jd_times, + 'mode': 'lm', + 'use_impactparameter_rather_than_inclination_to_fit': + use_impactparameter_rather_than_inclination_to_fit, + } + add_exposure_times_to_lc_fitter_kwargs(prefit_kwargs, good_exposure_times) + prefit = lc_fitter( + good_times, + good_flux, + good_unc, + good_airmass, + prior, + bounds, + **prefit_kwargs, + ) + if ( + run_final_fit_phase_residual_clip + and prefit is not None + and hasattr(prefit, 'residuals') + and hasattr(prefit, 'phase') + and np.shape(prefit.residuals) == np.shape(good_times) + and np.shape(prefit.phase) == np.shape(good_times) + ): + phase_clip_mask = phase_bin_sigma_clip(prefit.residuals, prefit.phase, sigma=3, bins=10) + min_required_points = max(len(bounds) + 1, LIGHTCURVE_MIN_VALID_POINTS) + if np.any(phase_clip_mask) and np.count_nonzero(~phase_clip_mask) >= min_required_points: + debug_phase_clip_keep_mask = np.asarray(~phase_clip_mask, dtype=bool).copy() + result['filter_diagnostics'].append(build_time_rejection_diagnostic( + "Final-fit phase residual clip", + good_times, + ~phase_clip_mask, + note="Dropped phase-binned residual outliers before the comparison-candidate ultranest fit.", + )) + good_times = good_times[~phase_clip_mask] + good_flux = good_flux[~phase_clip_mask] + good_unc = good_unc[~phase_clip_mask] + good_airmass = good_airmass[~phase_clip_mask] + good_jd_times = good_jd_times[~phase_clip_mask] + if good_exposure_times is not None: + good_exposure_times = good_exposure_times[~phase_clip_mask] + good_target_flux = good_target_flux[~phase_clip_mask] + good_comp_flux = good_comp_flux[~phase_clip_mask] + good_target_flux_error = good_target_flux_error[~phase_clip_mask] + good_comp_flux_error = good_comp_flux_error[~phase_clip_mask] + source_indices = source_indices[~phase_clip_mask] + + full_good_times = np.asarray(good_times, dtype=float) + full_good_flux = np.asarray(good_flux, dtype=float) + full_good_unc = np.asarray(good_unc, dtype=float) + full_good_airmass = np.asarray(good_airmass, dtype=float) + full_good_jd_times = np.asarray(good_jd_times, dtype=float) + full_good_exposure_times = ( + None if good_exposure_times is None else np.asarray(good_exposure_times, dtype=float) + ) + full_good_target_flux = np.asarray(good_target_flux, dtype=float) + full_good_comp_flux = np.asarray(good_comp_flux, dtype=float) + full_good_target_flux_error = np.asarray(good_target_flux_error, dtype=float) + full_good_comp_flux_error = np.asarray(good_comp_flux_error, dtype=float) + full_source_indices = np.asarray(source_indices, dtype=int) + + fast_binning = {'applied': False, 'note': None} + fit_times = full_good_times + fit_flux = full_good_flux + fit_unc = full_good_unc + fit_airmass = full_good_airmass + fit_jd_times = full_good_jd_times + fit_exposure_times = full_good_exposure_times + if run_fast_ultranest_before_final_run: + fast_binning = build_fast_ultranest_lightcurve_series( + full_good_times, + full_good_flux, + full_good_unc, + full_good_airmass, + jd_times=full_good_jd_times, + exposure_times_seconds=full_good_exposure_times, + ) + if fast_binning.get('applied'): + log_info(fast_binning['note']) + fit_times = fast_binning['time'] + fit_flux = fast_binning['flux'] + fit_unc = fast_binning['unc'] + fit_airmass = fast_binning['airmass'] + fit_jd_times = fast_binning['jd_times'] + fit_exposure_times = fast_binning.get('exposure_times_seconds') + + fit_prior = dict(prior) + fit_bounds = clone_lightcurve_bounds(bounds) + ensure_pre_final_ultranest_baseline_bounds(fit_prior, fit_bounds, fit_flux, fit_a2=True) + pre_ultranest_coverage_assessment = build_expected_transit_coverage_assessment( + full_good_times, + prior, + flux_values=full_good_flux, + flux_errors=full_good_unc, + tmid_search_summary=tmid_search_summary, + duration_prior=build_single_transit_duration_prior(p_dict), + ) + + final_fit, fitted_flux, fitted_unc = fit_final_lightcurve_with_oot_baseline_detrending( + fit_times, + fit_flux, + fit_unc, + fit_airmass, + fit_prior, + fit_bounds, + jd_times=fit_jd_times, + exposure_times_seconds=fit_exposure_times, + skip_airmass_fit=skip_final_airmass_fit, + airmass_skip_note=airmass_skip_note, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + detrend_on_outoftransit_baseline=detrend_on_outoftransit_baseline, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + plot_time_range=plot_time_range, + baseline_duration_multiplier=baseline_duration_multiplier, + expected_planet_dict=p_dict, + expected_tmid_search_summary=tmid_search_summary, + eebls_search_summary=eebls_search_summary, + extend_sparse_posterior_live_points=False, + keep_ultranest_sampler_for_deferred_extension=not bool(fast_binning.get('applied')), + fix_baseline_terms_for_final=not bool(fast_binning.get('applied')), + pre_ultranest_coverage_assessment=pre_ultranest_coverage_assessment, + search_restriction_prior=search_restriction_prior, + ) + annotate_fast_ultranest_binning(final_fit, fast_binning) + if final_fit is None: + result['failure_reason'] = "the full comparison-candidate reduction did not converge." + return result + + final_fit_times = np.asarray(getattr(final_fit, 'time', good_times), dtype=float) + if not fast_binning.get('applied') and ( + final_fit_times.shape != good_times.shape + or not np.allclose(final_fit_times, good_times, rtol=1e-10, atol=1e-10) + ): + final_time_indices = match_time_subset_indices(good_times, final_fit_times) + if final_time_indices is not None: + good_times = good_times[final_time_indices] + good_flux = good_flux[final_time_indices] + good_unc = good_unc[final_time_indices] + good_airmass = good_airmass[final_time_indices] + good_jd_times = good_jd_times[final_time_indices] + if good_exposure_times is not None: + good_exposure_times = good_exposure_times[final_time_indices] + good_target_flux = good_target_flux[final_time_indices] + good_comp_flux = good_comp_flux[final_time_indices] + good_target_flux_error = good_target_flux_error[final_time_indices] + good_comp_flux_error = good_comp_flux_error[final_time_indices] + source_indices = source_indices[final_time_indices] + + if run_final_residual_rejection and not fast_binning.get('applied'): + residual_payload = initialize_final_residual_rejection_payload( + enabled=True, + input_point_count=int(good_times.shape[0]), + ) + residual_stop_reason = None + for residual_refit_iteration in range(1, FINAL_RESIDUAL_REJECTION_MAX_REFITS + 1): + min_required_points = max(len(fit_bounds) + 1, LIGHTCURVE_MIN_VALID_POINTS) + residual_keep_mask, residual_summary = final_residual_rejection_keep_mask( + final_fit, + min_required_points=min_required_points, + ) + cycle_payload = build_final_residual_rejection_payload( + final_fit, + residual_keep_mask, + residual_summary, + source_indices=source_indices, + ) + if residual_keep_mask.shape != good_times.shape: + residual_stop_reason = 'shape_mismatch' + if int(residual_payload.get('rejected_point_count', 0) or 0) == 0: + residual_payload['note'] = ( + "Skipped; the final-fit residual array did not align with the retained light-curve points." + ) + else: + log_info( + "Warning: final residual rejection stopped because the residual array no longer " + "aligned with the retained light-curve points.", + warn=True, + ) + break + + if not residual_summary.get('applied'): + if int(residual_payload.get('rejected_point_count', 0) or 0) == 0: + residual_payload = dict(cycle_payload) + residual_payload.setdefault('refit_iteration_count', 0) + residual_payload.setdefault('refit_iterations', []) + else: + residual_payload = update_final_residual_rejection_final_pass( + residual_payload, + residual_summary, + ) + break + + clipped_times = good_times[residual_keep_mask] + clipped_flux = good_flux[residual_keep_mask] + clipped_unc = good_unc[residual_keep_mask] + clipped_airmass = good_airmass[residual_keep_mask] + clipped_jd_times = good_jd_times[residual_keep_mask] + clipped_exposure_times = ( + None if good_exposure_times is None else good_exposure_times[residual_keep_mask] + ) + clipped_target_flux = good_target_flux[residual_keep_mask] + clipped_comp_flux = good_comp_flux[residual_keep_mask] + clipped_target_flux_error = good_target_flux_error[residual_keep_mask] + clipped_comp_flux_error = good_comp_flux_error[residual_keep_mask] + clipped_source_indices = source_indices[residual_keep_mask] + + residual_refit_prior = dict(fit_prior) + final_parameters = getattr(final_fit, 'parameters', {}) + if isinstance(final_parameters, dict): + for key in ('rprs', 'ars', 'tmid', 'inc', 'a0', 'a1', 'a2'): + if key in residual_refit_prior and key in final_parameters: + residual_refit_prior[key] = final_parameters[key] + residual_refit_bounds = get_posterior_refit_final_bounds(final_fit, fit_bounds) + residual_coverage_assessment = build_expected_transit_coverage_assessment( + clipped_times, + residual_refit_prior, + flux_values=clipped_flux, + flux_errors=clipped_unc, + tmid_search_summary=build_ephemeris_tmid_search_summary_for_coverage( + clipped_times, + p_dict, + prior=residual_refit_prior, + duration_prior=build_single_transit_duration_prior(p_dict), + sigma_multiplier=35.0, + ), + duration_prior=build_single_transit_duration_prior(p_dict), + ) + refit, refit_flux, refit_unc = fit_final_lightcurve_with_oot_baseline_detrending( + clipped_times, + clipped_flux, + clipped_unc, + clipped_airmass, + residual_refit_prior, + residual_refit_bounds, + jd_times=clipped_jd_times, + exposure_times_seconds=clipped_exposure_times, + skip_airmass_fit=skip_final_airmass_fit, + airmass_skip_note=airmass_skip_note, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + detrend_on_outoftransit_baseline=detrend_on_outoftransit_baseline, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + plot_time_range=plot_time_range, + baseline_duration_multiplier=baseline_duration_multiplier, + expected_planet_dict=p_dict, + expected_tmid_search_summary=tmid_search_summary, + eebls_search_summary=eebls_search_summary, + extend_sparse_posterior_live_points=False, + keep_ultranest_sampler_for_deferred_extension=True, + fix_baseline_terms_for_final=True, + pre_ultranest_coverage_assessment=residual_coverage_assessment, + search_restriction_prior=build_search_restriction_prior_from_planet_dict(p_dict), + ) + if refit is None: + residual_stop_reason = 'refit_failed' + if int(residual_payload.get('rejected_point_count', 0) or 0) == 0: + residual_payload['note'] = ( + "Skipped; the residual-rejected final UltraNest refit did not converge, " + "so the unrejected final fit was retained." + ) + log_info( + "Warning: the residual-rejected final UltraNest refit did not converge; " + "retaining the last successful final fit.", + warn=True, + ) + break + + cycle_note = ( + f"Iteration {residual_refit_iteration}: {residual_summary.get('note')} " + f"Reran UltraNest on {int(clipped_times.shape[0])} point(s)." + ) + diagnostic = build_time_rejection_diagnostic( + f"Final residual rejection refit {residual_refit_iteration}", + good_times, + residual_keep_mask, + note=cycle_note, + ) + if diagnostic is not None: + result['filter_diagnostics'].append(diagnostic) + log_info(cycle_note) + residual_payload = record_final_residual_rejection_refit_cycle( + residual_payload, + cycle_payload, + residual_refit_iteration, + ) + annotate_fast_ultranest_binning(refit, fast_binning) + final_fit = refit + fitted_flux = np.asarray(refit_flux, dtype=float) + fitted_unc = np.asarray(refit_unc, dtype=float) + good_times = clipped_times + good_flux = clipped_flux + good_unc = clipped_unc + good_airmass = clipped_airmass + good_jd_times = clipped_jd_times + good_exposure_times = clipped_exposure_times + good_target_flux = clipped_target_flux + good_comp_flux = clipped_comp_flux + good_target_flux_error = clipped_target_flux_error + good_comp_flux_error = clipped_comp_flux_error + source_indices = clipped_source_indices + else: + residual_stop_reason = 'max_refits' + + residual_payload = finalize_final_residual_rejection_payload( + residual_payload, + current_point_count=int(good_times.shape[0]), + stopped_reason=residual_stop_reason, + ) + annotate_final_residual_rejection(final_fit, residual_payload) + else: + annotate_final_residual_rejection( + final_fit, + { + 'enabled': bool(run_final_residual_rejection), + 'applied': False, + 'note': ( + "Deferred to the selected full-resolution final refit." + if fast_binning.get('applied') + else "Disabled per optional_info setting." + ), + 'input_point_count': int(final_fit_times.size), + 'kept_point_count': int(final_fit_times.size), + 'rejected_point_count': 0, + 'sigma': FINAL_RESIDUAL_REJECTION_SIGMA, + }, + ) + + annotate_lightcurve_filter_diagnostics(final_fit, result['filter_diagnostics']) + annotate_selected_photometry_debug( + final_fit, + prepared['debug_times'], + prepared['debug_target_flux'], + prepared['debug_comp_flux'], + prepared['debug_raw_ratio'], + prepared['initial_sigma_keep_mask'], + target_flux_error=prepared.get('debug_target_flux_error'), + comp_flux_error=prepared.get('debug_comp_flux_error'), + relative_flux_error=prepared.get('debug_relative_flux_error'), + prefit_raw_ratio_keep_mask=prepared.get('prefit_raw_ratio_keep_mask'), + phase_clip_keep_mask_on_sigma_filtered=debug_phase_clip_keep_mask, + ) + + data_highres, duration_samples = estimate_transit_duration_samples_from_fit(final_fit) + result.update({ + 'applied': True, + 'fit': final_fit, + 'good_times': full_good_times if fast_binning.get('applied') else np.asarray(good_times, dtype=float), + 'good_flux': full_good_flux if fast_binning.get('applied') else np.asarray(good_flux, dtype=float), + 'good_unc': full_good_unc if fast_binning.get('applied') else np.asarray(good_unc, dtype=float), + 'good_airmass': full_good_airmass if fast_binning.get('applied') else np.asarray(good_airmass, dtype=float), + 'good_jd_times': full_good_jd_times if fast_binning.get('applied') else np.asarray(good_jd_times, dtype=float), + 'good_exposure_times_seconds': ( + full_good_exposure_times + if fast_binning.get('applied') + else None if good_exposure_times is None else np.asarray(good_exposure_times, dtype=float) + ), + 'good_target_flux': full_good_target_flux if fast_binning.get('applied') else np.asarray(good_target_flux, dtype=float), + 'good_comp_flux': full_good_comp_flux if fast_binning.get('applied') else np.asarray(good_comp_flux, dtype=float), + 'good_target_flux_error': full_good_target_flux_error if fast_binning.get('applied') else np.asarray(good_target_flux_error, dtype=float), + 'good_comp_flux_error': full_good_comp_flux_error if fast_binning.get('applied') else np.asarray(good_comp_flux_error, dtype=float), + 'source_indices': full_source_indices if fast_binning.get('applied') else np.asarray(source_indices, dtype=int), + 'fast_ultranest_binning': fast_binning, + 'fast_fit_good_times': np.asarray(fit_times, dtype=float), + 'fast_fit_good_flux': np.asarray(fitted_flux, dtype=float), + 'fast_fit_good_unc': np.asarray(fitted_unc, dtype=float), + 'fast_fit_good_airmass': np.asarray(fit_airmass, dtype=float), + 'fast_fit_good_jd_times': None if fit_jd_times is None else np.asarray(fit_jd_times, dtype=float), + 'fast_fit_good_exposure_times_seconds': ( + None if fit_exposure_times is None else np.asarray(fit_exposure_times, dtype=float) + ), + 'fast_fit_prior': fit_prior, + 'fast_fit_bounds': fit_bounds, + 'skip_airmass_fit': skip_final_airmass_fit, + 'airmass_skip_note': airmass_skip_note, + 'data_highres': data_highres, + 'duration_samples': duration_samples, + 'failure_reason': None, + 'note': 'completed the full comparison-candidate reduction directly from the raw target/reference light curve.', + }) + return result + + +def selected_final_live_point_target(enabled=None): + if enabled is None: + enabled = should_use_sparse_posterior_live_point_retry( + os.environ.get( + SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_ENV, + SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_DEFAULT, + ) + ) + base_live_points = get_configured_ultranest_min_num_live_points() + if not enabled: + return base_live_points, None + extension_factor = int(max(1, SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT)) + target_live_points = int(max( + base_live_points + extension_factor * base_live_points, + base_live_points + 1, + )) + return base_live_points, target_live_points + + +def coerce_fixed_baseline_error(value, default=None): + try: + value = float(value) + except (TypeError, ValueError): + return default + if np.isfinite(value) and value >= 0: + return value + return default + + +def baseline_fixed_errors_from_fit(fit): + parameters = getattr(fit, 'parameters', {}) if fit is not None else {} + errors = getattr(fit, 'errors', {}) if fit is not None else {} + fixed_errors = {} + if isinstance(errors, dict): + for key in ('a0', 'a1', 'a2'): + value = coerce_fixed_baseline_error(errors.get(key)) + if value is not None: + fixed_errors[key] = value + if 'a0' in fixed_errors and 'a1' not in fixed_errors: + fixed_errors['a1'] = fixed_errors['a0'] + if ( + isinstance(parameters, dict) + and 'a2' in parameters + and 'a2' not in fixed_errors + ): + a2_error = estimate_fixed_airmass_coefficient_error( + getattr(fit, 'data', None), + getattr(fit, 'dataerr', None), + getattr(fit, 'airmass', None), + ) + if a2_error is not None: + fixed_errors['a2'] = a2_error + return fixed_errors + + +def baseline_fixed_errors_from_oot_parameter_result(result): + fixed_errors = {} + if not isinstance(result, dict): + return fixed_errors + + a0_error = coerce_fixed_baseline_error(result.get('a0_error')) + if a0_error is not None: + fixed_errors['a0'] = a0_error + fixed_errors['a1'] = a0_error + + a2_error = coerce_fixed_baseline_error(result.get('a2_error')) + if a2_error is not None: + fixed_errors['a2'] = a2_error + + return fixed_errors + + +def build_full_resolution_final_prior_from_previous_fit(previous_fit, p_dict): + previous_parameters = getattr(previous_fit, 'parameters', {}) + if not isinstance(previous_parameters, dict): + previous_parameters = {} + + prior = { + 'rprs': p_dict.get('rprs', previous_parameters.get('rprs')), + 'ars': p_dict.get('aRs', previous_parameters.get('ars')), + 'per': p_dict.get('pPer', previous_parameters.get('per')), + 'inc': p_dict.get('inc', previous_parameters.get('inc')), + 'u0': previous_parameters.get('u0', 0.0), + 'u1': previous_parameters.get('u1', 0.0), + 'u2': previous_parameters.get('u2', 0.0), + 'u3': previous_parameters.get('u3', 0.0), + 'ecc': p_dict.get('ecc', previous_parameters.get('ecc', 0.0)), + 'omega': p_dict.get('omega', previous_parameters.get('omega', 0.0)), + 'tmid': p_dict.get('midT', previous_parameters.get('tmid')), + 'a2': previous_parameters.get('a2', 0.0), + 'a0': previous_parameters.get('a0', previous_parameters.get('a1', 1.0)), + } + prior['a1'] = previous_parameters.get('a1', prior['a0']) + prior.update(previous_parameters) + if 'a0' not in prior and 'a1' in prior: + prior['a0'] = prior['a1'] + if 'a1' not in prior and 'a0' in prior: + prior['a1'] = prior['a0'] + return prior + + +def refit_selected_fast_comparison_on_full_lightcurve( + selected_result, + p_dict, + skip_airmass_fit=False, + airmass_skip_note=None, + detrend_on_outoftransit_baseline=True, + oot_baseline_min_points_per_side=OUT_OF_TRANSIT_BASELINE_MIN_SIDE_POINTS_DEFAULT, + use_impactparameter_rather_than_inclination_to_fit=True, + plot_time_range=None, + duration_prior=None, + sparse_live_point_extension_enabled=None, + run_final_residual_rejection=FINAL_RESIDUAL_REJECTION_DEFAULT, +): + previous_fit = selected_result.get('fit') if isinstance(selected_result, dict) else None + if previous_fit is None or not getattr(previous_fit, 'fast_ultranest_binning_applied', False): + return None + + times = np.asarray(selected_result.get('good_times'), dtype=float) + flux_values = np.asarray(selected_result.get('good_flux'), dtype=float) + flux_errors = np.asarray(selected_result.get('good_unc'), dtype=float) + airmass = np.asarray(selected_result.get('good_airmass'), dtype=float) + jd_times = selected_result.get('good_jd_times') + jd_times = None if jd_times is None else np.asarray(jd_times, dtype=float) + exposure_times = selected_result.get('good_exposure_times_seconds') + exposure_times = None if exposure_times is None else np.asarray(exposure_times, dtype=float) + if not (times.shape == flux_values.shape == flux_errors.shape == airmass.shape): + log_info( + "Warning: Could not run the full-resolution selected comparison-star final fit " + "because the saved fast-fit light-curve arrays were not aligned.", + warn=True, + ) + return None + if jd_times is not None and jd_times.shape != times.shape: + jd_times = None + if exposure_times is not None and exposure_times.shape != times.shape: + exposure_times = None + + original_times = times.copy() + base_filter_diagnostics = [ + dict(diagnostic) + for diagnostic in getattr(previous_fit, 'frame_filter_diagnostics', []) + if isinstance(diagnostic, dict) + ] + residual_rejection_diagnostics = [] + residual_rejection_payload = None + + def aligned_selected_array(key, dtype=float): + values = selected_result.get(key) + if values is None: + return None + array = np.asarray(values, dtype=dtype).reshape(-1) + return array if array.shape == times.shape else None + + target_flux_values = aligned_selected_array('good_target_flux') + comp_flux_values = aligned_selected_array('good_comp_flux') + target_flux_error_values = aligned_selected_array('good_target_flux_error') + comp_flux_error_values = aligned_selected_array('good_comp_flux_error') + source_indices = aligned_selected_array('source_indices', dtype=int) + + prior = build_full_resolution_final_prior_from_previous_fit(previous_fit, p_dict) + search_restriction_prior = build_search_restriction_prior_from_planet_dict(p_dict) + fallback_bounds = selected_result.get('fast_fit_bounds') + if not isinstance(fallback_bounds, dict): + fallback_bounds = getattr(previous_fit, 'bounds', {}) + bounds = get_posterior_refit_final_bounds(previous_fit, fallback_bounds) + bounds = clone_lightcurve_bounds(bounds) + for key in ('a0', 'a1', 'a2'): + bounds.pop(key, None) + + for key in ('rprs', 'tmid', 'ars', 'inc'): + if key not in bounds: + if key == 'rprs': + bounds[key] = build_initial_rprs_bounds(prior.get('rprs', p_dict.get('rprs', 0.1))) + elif key == 'tmid': + tmid = prior.get('tmid', p_dict.get('midT', np.nan)) + tmid_unc = p_dict.get('midTUnc', 0.01) + try: + half_width = max(float(tmid_unc) * 3.0, np.finfo(float).eps) + except (TypeError, ValueError): + half_width = 0.01 + bounds[key] = [float(tmid) - half_width, float(tmid) + half_width] + elif key == 'ars': + bounds[key] = build_initial_ars_bounds( + prior.get('ars', p_dict.get('aRs')), + p_dict.get('aRsUnc'), + search_restriction_prior=search_restriction_prior, + ) + elif key == 'inc': + inc = float(prior.get('inc', p_dict.get('inc', 89.0))) + bounds[key] = [inc - 5.0, min(90.0, inc + 5.0)] + + fit_flux = flux_values + fit_unc = flux_errors + detrend_result = {'applied': False, 'note': 'Disabled; using the full-resolution light curve directly.'} + if detrend_on_outoftransit_baseline: + detrend_result = detrend_flux_on_out_of_transit_baseline( + times, + flux_values, + flux_errors, + previous_fit, + prior=prior, + min_side_points=oot_baseline_min_points_per_side, + ) + if detrend_result.get('applied'): + fit_flux = np.asarray(detrend_result['flux'], dtype=float) + fit_unc = np.asarray(detrend_result['unc'], dtype=float) + prior['a0'] = 1.0 + prior['a1'] = 1.0 + prior['a2'] = 0.0 + log_info( + "Applying selected full-resolution out-of-transit linear baseline detrending: " + f"{detrend_result.get('note', 'baseline fit details unavailable')}" + ) + else: + log_info( + "Selected full-resolution out-of-transit baseline detrending skipped: " + f"{detrend_result.get('note', 'baseline fit details unavailable')}" + ) + + search_restriction_prior = enrich_search_restriction_prior_with_rprs_data_uncertainty( + search_restriction_prior, + times, + fit_flux, + fit_unc, + prior, + context_label="selected full-resolution final light curve", + ) + bounds = widen_rprs_bounds_to_data_uncertainty_window(bounds, search_restriction_prior) + + fixed_errors = baseline_fixed_errors_from_fit(previous_fit) + if detrend_result.get('applied'): + fixed_errors = { + 'a0': fixed_errors.get('a0', 0.0), + 'a1': fixed_errors.get('a1', fixed_errors.get('a0', 0.0)), + 'a2': fixed_errors.get('a2', 0.0), + } + base_live_points, target_live_points = selected_final_live_point_target( + sparse_live_point_extension_enabled, + ) + min_live_points = target_live_points if target_live_points is not None else base_live_points + coverage_duration_prior = ( + duration_prior if isinstance(duration_prior, dict) else build_single_transit_duration_prior(p_dict) + ) + pre_ultranest_coverage_assessment = build_expected_transit_coverage_assessment( + times, + prior, + flux_values=fit_flux, + flux_errors=fit_unc, + tmid_search_summary=build_ephemeris_tmid_search_summary_for_coverage( + times, + p_dict, + prior=prior, + duration_prior=coverage_duration_prior, + sigma_multiplier=35.0, + ), + duration_prior=coverage_duration_prior, + ) + log_expected_transit_coverage_assessment(pre_ultranest_coverage_assessment) + + fixed_baseline_source = ( + "with a flat fixed baseline after out-of-transit detrending" + if detrend_result.get('applied') + else "with fixed a0/a2 from the previous fast UltraNest fit" + ) + log_info( + "Running the selected comparison-star final UltraNest fit on the full-resolution light curve " + f"{fixed_baseline_source} at {min_live_points} minimum live points." + ) + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + times, + fit_flux, + fit_unc, + airmass, + prior, + bounds, + jd_times=jd_times, + exposure_times_seconds=exposure_times, + use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + max_rprs_retries=0, + max_ars_retries=0, + max_impact_parameter_retries=0, + duration_prior=duration_prior, + keep_ultranest_sampler=False, + fixed_parameter_errors=fixed_errors, + fixed_flux_baseline=True, + ultranest_min_num_live_points=min_live_points, + pre_ultranest_coverage_assessment=pre_ultranest_coverage_assessment, + search_restriction_prior=search_restriction_prior, + ) + if fit is None: + return None + + if run_final_residual_rejection: + residual_rejection_payload = initialize_final_residual_rejection_payload( + enabled=True, + input_point_count=int(times.shape[0]), + ) + residual_stop_reason = None + for residual_refit_iteration in range(1, FINAL_RESIDUAL_REJECTION_MAX_REFITS + 1): + min_required_points = max(len(bounds) + 1, LIGHTCURVE_MIN_VALID_POINTS) + residual_keep_mask, residual_summary = final_residual_rejection_keep_mask( + fit, + min_required_points=min_required_points, + ) + cycle_payload = build_final_residual_rejection_payload( + fit, + residual_keep_mask, + residual_summary, + source_indices=source_indices, + ) + if residual_keep_mask.shape != times.shape: + residual_stop_reason = 'shape_mismatch' + if int(residual_rejection_payload.get('rejected_point_count', 0) or 0) == 0: + residual_rejection_payload['note'] = ( + "Skipped; the final-fit residual array did not align with the retained light-curve points." + ) + else: + log_info( + "Warning: final residual rejection stopped because the residual array no longer " + "aligned with the retained light-curve points.", + warn=True, + ) + break + + if not residual_summary.get('applied'): + if int(residual_rejection_payload.get('rejected_point_count', 0) or 0) == 0: + residual_rejection_payload = dict(cycle_payload) + residual_rejection_payload.setdefault('refit_iteration_count', 0) + residual_rejection_payload.setdefault('refit_iterations', []) + else: + residual_rejection_payload = update_final_residual_rejection_final_pass( + residual_rejection_payload, + residual_summary, + ) + break + + retained_times = times[residual_keep_mask] + retained_flux = fit_flux[residual_keep_mask] + retained_unc = fit_unc[residual_keep_mask] + retained_airmass = airmass[residual_keep_mask] + retained_jd_times = None if jd_times is None else jd_times[residual_keep_mask] + retained_exposure_times = None if exposure_times is None else exposure_times[residual_keep_mask] + retained_target_flux_values = ( + None if target_flux_values is None else target_flux_values[residual_keep_mask] + ) + retained_comp_flux_values = ( + None if comp_flux_values is None else comp_flux_values[residual_keep_mask] + ) + retained_target_flux_error_values = ( + None if target_flux_error_values is None else target_flux_error_values[residual_keep_mask] + ) + retained_comp_flux_error_values = ( + None if comp_flux_error_values is None else comp_flux_error_values[residual_keep_mask] + ) + retained_source_indices = ( + None if source_indices is None else source_indices[residual_keep_mask] + ) + + residual_refit_prior = dict(prior) + final_parameters = getattr(fit, 'parameters', {}) + if isinstance(final_parameters, dict): + for key in ('rprs', 'ars', 'tmid', 'inc'): + if key in residual_refit_prior and key in final_parameters: + residual_refit_prior[key] = final_parameters[key] + residual_refit_bounds = get_posterior_refit_final_bounds(fit, bounds) + residual_search_restriction_prior = enrich_search_restriction_prior_with_rprs_data_uncertainty( + search_restriction_prior, + retained_times, + retained_flux, + retained_unc, + residual_refit_prior, + context_label="residual-rejected selected final light curve", + ) + residual_refit_bounds = widen_rprs_bounds_to_data_uncertainty_window( + residual_refit_bounds, + residual_search_restriction_prior, + ) + residual_coverage_assessment = build_expected_transit_coverage_assessment( + retained_times, + residual_refit_prior, + flux_values=retained_flux, + flux_errors=retained_unc, + tmid_search_summary=build_ephemeris_tmid_search_summary_for_coverage( + retained_times, + p_dict, + prior=residual_refit_prior, + duration_prior=coverage_duration_prior, + sigma_multiplier=35.0, + ), + duration_prior=coverage_duration_prior, + ) + log_expected_transit_coverage_assessment(residual_coverage_assessment) + refit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + retained_times, + retained_flux, + retained_unc, + retained_airmass, + residual_refit_prior, + residual_refit_bounds, + jd_times=retained_jd_times, + exposure_times_seconds=retained_exposure_times, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + max_rprs_retries=0, + max_ars_retries=0, + max_impact_parameter_retries=0, + duration_prior=duration_prior, + keep_ultranest_sampler=False, + fixed_parameter_errors=fixed_errors, + fixed_flux_baseline=True, + ultranest_min_num_live_points=min_live_points, + pre_ultranest_coverage_assessment=residual_coverage_assessment, + search_restriction_prior=residual_search_restriction_prior, + ) + if refit is None: + residual_stop_reason = 'refit_failed' + if int(residual_rejection_payload.get('rejected_point_count', 0) or 0) == 0: + residual_rejection_payload['note'] = ( + "Skipped; the residual-rejected selected full-resolution UltraNest refit " + "did not converge, so the unrejected final fit was retained." + ) + log_info( + "Warning: the residual-rejected selected full-resolution UltraNest refit did not " + "converge; retaining the last successful final fit.", + warn=True, + ) + break + + cycle_note = ( + f"Iteration {residual_refit_iteration}: {residual_summary.get('note')} " + f"Reran UltraNest on {int(retained_times.shape[0])} point(s)." + ) + residual_rejection_diagnostic = build_time_rejection_diagnostic( + f"Final residual rejection refit {residual_refit_iteration}", + times, + residual_keep_mask, + note=cycle_note, + ) + if residual_rejection_diagnostic is not None: + residual_rejection_diagnostics.append(residual_rejection_diagnostic) + log_info(cycle_note) + residual_rejection_payload = record_final_residual_rejection_refit_cycle( + residual_rejection_payload, + cycle_payload, + residual_refit_iteration, + ) + fit = refit + times = retained_times + fit_flux = retained_flux + fit_unc = retained_unc + airmass = retained_airmass + jd_times = retained_jd_times + exposure_times = retained_exposure_times + target_flux_values = retained_target_flux_values + comp_flux_values = retained_comp_flux_values + target_flux_error_values = retained_target_flux_error_values + comp_flux_error_values = retained_comp_flux_error_values + source_indices = retained_source_indices + pre_ultranest_coverage_assessment = residual_coverage_assessment + else: + residual_stop_reason = 'max_refits' + + residual_rejection_payload = finalize_final_residual_rejection_payload( + residual_rejection_payload, + current_point_count=int(times.shape[0]), + stopped_reason=residual_stop_reason, + ) + else: + residual_rejection_payload = { + 'enabled': False, + 'applied': False, + 'note': "Disabled per optional_info setting.", + 'input_point_count': int(times.shape[0]), + 'kept_point_count': int(times.shape[0]), + 'rejected_point_count': 0, + 'sigma': FINAL_RESIDUAL_REJECTION_SIGMA, + } + + annotate_pre_ultranest_transit_coverage(fit, pre_ultranest_coverage_assessment) + fit = apply_plot_time_range(fit, original_times if plot_time_range is None else plot_time_range) + annotate_airmass_fit(fit, airmass, skip_airmass_fit, note=airmass_skip_note) + baseline_parameter_note = ( + "Full-resolution out-of-transit linear detrending flattened the final-fit light curve; " + "fixed the final baseline to a0=1 and a2=0 instead of reusing the previous fast-fit baseline scale." + if detrend_result.get('applied') + else "Used a0 and a2 from the previous fast UltraNest fit for the full-resolution final run." + ) + annotate_out_of_transit_baseline_parameter_fit( + fit, + not bool(detrend_result.get('applied')), + note=baseline_parameter_note, + pre_points=0, + post_points=0, + a0=prior.get('a0'), + a0_error=fixed_errors.get('a0'), + a2=prior.get('a2'), + a2_error=fixed_errors.get('a2'), + ) + if detrend_result.get('applied'): + previous_parameters = getattr(previous_fit, 'parameters', {}) + previous_parameters = previous_parameters if isinstance(previous_parameters, dict) else {} + previous_errors = getattr(previous_fit, 'errors', {}) + previous_errors = previous_errors if isinstance(previous_errors, dict) else {} + scale_parameter = 'a1' if 'a1' in previous_parameters else 'a0' + annotate_pre_detrending_baseline_coefficients( + fit, + source="selected fast UltraNest fit before out-of-transit linear baseline detrending", + scale_parameter=scale_parameter, + scale_value=previous_parameters.get(scale_parameter), + scale_error=previous_errors.get(scale_parameter, fixed_errors.get(scale_parameter)), + a2_value=previous_parameters.get('a2'), + a2_error=previous_errors.get('a2', fixed_errors.get('a2')), + ) + annotate_out_of_transit_baseline_detrending( + fit, + bool(detrend_result.get('applied')), + note=detrend_result.get('note'), + slope=detrend_result.get('slope'), + intercept=detrend_result.get('intercept'), + reference_time_bjd_tdb=detrend_result.get('reference_time_bjd_tdb'), + pre_points=detrend_result.get('pre_points', 0), + post_points=detrend_result.get('post_points', 0), + ) + annotate_fast_ultranest_binning( + fit, + { + 'applied': False, + 'original_point_count': int(original_times.shape[0]), + 'binned_point_count': int(times.shape[0]), + 'note': 'Full-resolution selected comparison-star final run; fast binning was not applied.', + }, + ) + annotate_transit_qc_expected_values(fit, p_dict) + if target_live_points is not None: + diagnostics = evaluate_sparse_posterior_sample_support(fit, base_live_points=base_live_points) + annotate_sparse_posterior_live_point_extension( + fit, + True, + True, + note=( + "Applied full-resolution selected comparison-star final UltraNest run " + f"({base_live_points}->{target_live_points} minimum live points) " + f"{fixed_baseline_source}." + ), + diagnostics=diagnostics, + post_extension_diagnostics=diagnostics, + base_live_points=base_live_points, + target_live_points=target_live_points, + extension_factor=SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT, + ) + else: + annotate_sparse_posterior_live_point_extension(fit, False, False) + if residual_rejection_diagnostics: + base_filter_diagnostics.extend(residual_rejection_diagnostics) + annotate_lightcurve_filter_diagnostics(fit, base_filter_diagnostics) + annotate_final_residual_rejection(fit, residual_rejection_payload) + selected_debug = getattr(previous_fit, 'selected_photometry_debug', None) + if selected_debug is not None: + fit.selected_photometry_debug = copy.deepcopy(selected_debug) + selected_result['good_times'] = np.asarray(times, dtype=float) + selected_result['good_flux'] = np.asarray(fit_flux, dtype=float) + selected_result['good_unc'] = np.asarray(fit_unc, dtype=float) + selected_result['good_airmass'] = np.asarray(airmass, dtype=float) + selected_result['good_jd_times'] = None if jd_times is None else np.asarray(jd_times, dtype=float) + if target_flux_values is not None: + selected_result['good_target_flux'] = np.asarray(target_flux_values, dtype=float) + selected_result['tflux_fit'] = np.asarray(target_flux_values, dtype=float) + if comp_flux_values is not None: + selected_result['good_comp_flux'] = np.asarray(comp_flux_values, dtype=float) + selected_result['cflux_fit'] = np.asarray(comp_flux_values, dtype=float) + if target_flux_error_values is not None: + selected_result['good_target_flux_error'] = np.asarray(target_flux_error_values, dtype=float) + selected_result['tflux_fit_error'] = np.asarray(target_flux_error_values, dtype=float) + if comp_flux_error_values is not None: + selected_result['good_comp_flux_error'] = np.asarray(comp_flux_error_values, dtype=float) + selected_result['cflux_fit_error'] = np.asarray(comp_flux_error_values, dtype=float) + annotate_stellar_variability_raw_photometry( + fit, + selected_result.get('tflux_fit'), + selected_result.get('cflux_fit'), + target_flux_error=selected_result.get('tflux_fit_error'), + comp_flux_error=selected_result.get('cflux_fit_error'), + ) + if source_indices is not None: + selected_result['source_indices'] = np.asarray(source_indices, dtype=int) + annotate_transit_detection_qc(fit) + clear_fit_ultranest_resume_state(fit) + return fit, fit_flux, fit_unc + + +def save_comparison_candidate_full_reduction_outputs(save_dir, provisional_fit, final_fit, + p_dict, observation_date, comp_index, + comp_coords=None, min_aperture=None, min_annulus=None, + adaptive_summary=None, method_label=None, + selection_summary=None, duration_samples=None, + data_highres=None): + if save_dir is None or final_fit is None or observation_date is None or comp_index is None: + return None + + candidate_dir = comparison_candidate_output_dir(save_dir, comp_index) + temp_dir = candidate_dir / "working_artifacts" + temp_dir.mkdir(parents=True, exist_ok=True) + + archive_errors = [] + debug_series_path = None + bestfit_plot_path = None + triangle_plot_path = None + + debug_fit = provisional_fit if provisional_fit is not None else final_fit + if debug_fit is not None: + try: + debug_series_path = save_selected_photometry_debug_series( + candidate_dir, + p_dict['pName'], + observation_date, + debug_fit, + ) + except Exception as exc: + archive_errors.append(archive_exception_payload( + "Could not save the selected raw target/reference ratio diagnostics", + exc, + )) + + plotter = getattr(final_fit, 'plot_bestfit', None) + if callable(plotter): + try: + plot_kwargs = {} + if callable_accepts_keyword(plotter, 'show_flux_baseline_label'): + plot_kwargs['show_flux_baseline_label'] = False + fig, _ = plotter(**plot_kwargs) + bestfit_plot_path = temp_dir / safe_output_filename( + "BestFit", + p_dict['pName'], + filename_date_token(observation_date), + extension="png", + ) + fig.savefig(bestfit_plot_path) + plt.close(fig) + except Exception as exc: + archive_errors.append(archive_exception_payload( + "Could not save the final best-fit plot", + exc, + )) + + if callable(getattr(final_fit, 'plot_triangle', None)): + try: + fig = _plot_triangle_for_output( + final_fit, + plot_title=f"Comparison candidate #{int(comp_index) + 1} fit", + ) + triangle_plot_path = comparison_candidate_triangle_plot_output_path( + candidate_dir, + p_dict['pName'], + observation_date, + comp_index, + ) + if fig is not None: + fig.savefig(triangle_plot_path) + plt.close(fig) + except Exception as exc: + archive_errors.append(archive_exception_payload( + "Could not save the triangle plot", + exc, + )) + + duration_samples = np.asarray([] if duration_samples is None else duration_samples, dtype=float) + if duration_samples.size == 0: + measured_duration = getattr(final_fit, 'duration_measured', np.nan) + if np.isfinite(measured_duration) and measured_duration > 0: + duration_samples = np.asarray([measured_duration], dtype=float) + + candidate_info_dict = { + 'save': str(candidate_dir), + 'date': observation_date, + } + try: + if data_highres is None: + data_highres, _ = estimate_transit_duration_samples_from_fit(final_fit, sample_count=1) + if data_highres is not None: + plot_final_lightcurve( + final_fit, + data_highres, + p_dict['pName'], + candidate_info_dict['save'], + observation_date, + ) + plot_prior_posterior_comparison(final_fit, p_dict, p_dict['pName'], candidate_info_dict['save'], observation_date) + plot_ktmf_qc_metrics(final_fit, p_dict['pName'], candidate_info_dict['save'], observation_date) + except Exception as exc: + archive_errors.append(archive_exception_payload( + "Could not save the final lightcurve, prior/posterior comparison, or KTMF QC plot", + exc, + )) + + output_files = OutputFiles(final_fit, p_dict, candidate_info_dict, duration_samples) + try: + phase = get_phase(final_fit.time, p_dict['pPer'], final_fit.parameters['tmid']) + output_files.differential_magnitude() + output_files.stellar_variability_differential_magnitude() + output_files.final_lightcurve(phase) + except Exception as exc: + archive_errors.append(archive_exception_payload( + "Could not save FinalLightCurve CSV", + exc, + )) + + try: + output_files.final_planetary_params( + phot_opt=True, + vsp_params=[], + comp_star=int(comp_index + 1), + comp_coords=comp_coords, + min_aper=0 if min_aperture is None else np.round(min_aperture, 2), + min_annul=(None if min_annulus is None else np.round(min_annulus, 2)), + adaptive_summary=adaptive_summary, + ) + except Exception as exc: + archive_errors.append(archive_exception_payload( + "Could not save FinalParams JSON", + exc, + )) + + summary_path = temp_dir / safe_output_filename( + "ComparisonCandidateSummary", + p_dict['pName'], + filename_date_token(observation_date), + extension="json", + ) + summary_payload = { + 'planet_name': p_dict['pName'], + 'observation_date': observation_date, + 'comparison_star': int(comp_index + 1), + 'comparison_position': comp_coords, + 'method_label': method_label, + 'selection_summary': selection_summary or {}, + 'parameter_summary': summarize_lightcurve_fit_parameters(final_fit), + 'transit_qc': getattr(final_fit, 'transit_qc', None), + 'saved_debug_series': None if debug_series_path is None else str(debug_series_path), + 'saved_bestfit_plot': None if bestfit_plot_path is None else str(bestfit_plot_path), + 'saved_triangle_plot': None if triangle_plot_path is None else str(triangle_plot_path), + 'archive_errors': archive_errors, + } + with summary_path.open('w', encoding='utf-8') as handle: + json.dump(make_json_safe(summary_payload), handle, indent=4) + + return candidate_dir + + +def archive_failed_comparison_fit(save_dir, planet_name, observation_date, attempt, method_label=None): + if save_dir is None or planet_name is None or observation_date is None or not attempt: + return None + + comp_index = attempt.get('comp_index') + if comp_index is None: + return None + + archive_dir = failed_comparison_archive_dir(save_dir, comp_index) + temp_dir = archive_dir / "working_artifacts" + temp_dir.mkdir(parents=True, exist_ok=True) + + fit = attempt.get('fit') + archive_errors = [] + debug_series_path = None + bestfit_plot_path = None + + if fit is not None: + try: + debug_series_path = save_selected_photometry_debug_series( + archive_dir, + planet_name, + observation_date, + fit, + ) + except Exception as exc: + archive_errors.append(archive_exception_payload( + "Could not save the selected raw target/reference ratio diagnostics", + exc, + )) + + plotter = getattr(fit, 'plot_bestfit', None) + if callable(plotter): + try: + plot_kwargs = {} + if callable_accepts_keyword(plotter, 'show_flux_baseline_label'): + plot_kwargs['show_flux_baseline_label'] = False + fig, _ = plotter(**plot_kwargs) + bestfit_plot_path = temp_dir / safe_output_filename( + "BestFit", + planet_name, + filename_date_token(observation_date), + extension="png", + ) + fig.savefig(bestfit_plot_path) + plt.close(fig) + except Exception as exc: + archive_errors.append(archive_exception_payload( + "Could not save the provisional best-fit plot", + exc, + )) + + summary_path = temp_dir / safe_output_filename( + "FailedFitSummary", + planet_name, + filename_date_token(observation_date), + extension="json", + ) + summary_payload = { + 'planet_name': planet_name, + 'observation_date': observation_date, + 'comparison_star': None if comp_index is None else int(comp_index + 1), + 'comparison_label': attempt.get('label', f"Comp {comp_index + 1}"), + 'comparison_position': attempt.get('position'), + 'method_label': method_label, + 'failure_reason': attempt.get('failure_reason'), + 'fit_diagnostics': attempt.get('fit_diagnostics') or {}, + 'parameter_summary': attempt.get('parameter_summary'), + 'fit_point_count': attempt.get('fit_point_count'), + 'eebls_snr': attempt.get('eebls_snr'), + 'transit_delta_bic': attempt.get('transit_delta_bic'), + 'residual_scatter': attempt.get('residual_scatter'), + 'ktmf_metric': attempt.get('ktmf_metric'), + 'ktmf_contributions': attempt.get('ktmf_contributions') or [], + 'transit_qc': getattr(fit, 'transit_qc', None) if fit is not None else None, + 'saved_debug_series': None if debug_series_path is None else str(debug_series_path), + 'saved_bestfit_plot': None if bestfit_plot_path is None else str(bestfit_plot_path), + 'archive_errors': archive_errors, + } + with summary_path.open('w', encoding='utf-8') as handle: + json.dump(make_json_safe(summary_payload), handle, indent=4) + + return archive_dir + + +def save_selected_photometry_debug_series(save_dir, planet_name, observation_date, fit): + if fit is None: + return None + + debug = getattr(fit, 'selected_photometry_debug', None) + if not debug: + return None + + times = np.asarray(debug.get('times'), dtype=float) + target_flux = np.asarray(debug.get('target_flux'), dtype=float) + comp_flux = np.asarray(debug.get('comp_flux'), dtype=float) + raw_ratio = np.asarray(debug.get('raw_ratio'), dtype=float) + target_flux_error = np.asarray( + debug.get('target_flux_error', np.full(times.shape, np.nan)), + dtype=float, + ) + comp_flux_error = np.asarray( + debug.get('comp_flux_error', np.full(times.shape, np.nan)), + dtype=float, + ) + relative_flux_error = np.asarray( + debug.get('relative_flux_error', np.full(times.shape, np.nan)), + dtype=float, + ) + initial_sigma_keep_mask = np.asarray(debug.get('initial_sigma_keep_mask'), dtype=bool) + prefit_raw_ratio_keep_mask = np.asarray( + debug.get('prefit_raw_ratio_keep_mask', np.ones(initial_sigma_keep_mask.shape)), + dtype=bool, + ) + phase_clip_keep_mask = np.asarray( + debug.get( + 'phase_clip_keep_mask_on_sigma_filtered', + np.ones(np.count_nonzero(initial_sigma_keep_mask & prefit_raw_ratio_keep_mask)), + ), + dtype=bool, + ) + + if not ( + times.shape == target_flux.shape == comp_flux.shape == raw_ratio.shape == initial_sigma_keep_mask.shape + ): + return None + if target_flux_error.shape != times.shape: + target_flux_error = np.full(times.shape, np.nan, dtype=float) + if comp_flux_error.shape != times.shape: + comp_flux_error = np.full(times.shape, np.nan, dtype=float) + if relative_flux_error.shape != times.shape: + relative_flux_error = np.full(times.shape, np.nan, dtype=float) + if prefit_raw_ratio_keep_mask.shape != initial_sigma_keep_mask.shape: + prefit_raw_ratio_keep_mask = np.ones(initial_sigma_keep_mask.shape, dtype=bool) + + phase_keep_full = np.zeros(times.shape[0], dtype=bool) + prefit_kept_indices = np.flatnonzero(initial_sigma_keep_mask & prefit_raw_ratio_keep_mask) + if prefit_kept_indices.size: + if phase_clip_keep_mask.shape[0] != prefit_kept_indices.size: + phase_clip_keep_mask = np.ones(prefit_kept_indices.size, dtype=bool) + phase_keep_full[prefit_kept_indices] = phase_clip_keep_mask + + output_dir = Path(save_dir) / "working_artifacts" + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / safe_output_filename( + "SelectedPhotometryRawRatio", + planet_name, + filename_date_token(observation_date), + extension="csv", + ) + + output_rows = np.column_stack( + [ + times, + target_flux, + comp_flux, + raw_ratio, + target_flux_error, + comp_flux_error, + relative_flux_error, + initial_sigma_keep_mask.astype(int), + prefit_raw_ratio_keep_mask.astype(int), + phase_keep_full.astype(int), + ] + ) + np.savetxt( + output_path, + output_rows, + delimiter=",", + header=( + "BJD_TDB,Target Flux,Comp Flux,Raw Ratio," + "Target Flux Error,Comp Flux Error,Relative Flux Error," + "Kept After Initial Sigma Clip,Kept After Pre-Fit Raw Ratio Clip," + "Kept After Phase Residual Clip" + ), + comments="", + fmt=["%.8f", "%.8f", "%.8f", "%.8f", "%.8f", "%.8f", "%.8f", "%d", "%d", "%d"], + ) + return output_path + + +def annotate_rprs_posterior_refit(fit, applied, note=None, history=None): + annotate_parameter_posterior_refit(fit, 'rprs', applied, note=note, history=history) + + +def annotate_ars_posterior_refit(fit, applied, note=None, history=None): + annotate_parameter_posterior_refit(fit, 'ars', applied, note=note, history=history) + + +def annotate_impact_parameter_posterior_refit(fit, applied, note=None, history=None): + annotate_parameter_posterior_refit(fit, 'b', applied, note=note, history=history) + + +def annotate_parameter_posterior_refit(fit, parameter_key, applied, note=None, history=None): + if fit is None: + return + + history = [] if history is None else list(history) + attr_prefix = f"{parameter_key}_posterior_refit" + setattr(fit, f"{attr_prefix}_applied", bool(applied)) + setattr(fit, f"{attr_prefix}_note", note) + setattr(fit, f"{attr_prefix}_count", len(history)) + setattr(fit, f"{attr_prefix}_history", history) + if history: + latest = history[-1] + setattr(fit, f"{attr_prefix}_edge", latest.get('edge')) + setattr(fit, f"{attr_prefix}_mode", latest.get('mode')) + setattr(fit, f"{attr_prefix}_std", latest.get('std')) + setattr(fit, f"{attr_prefix}_original_bounds", latest.get('original_bounds')) + setattr(fit, f"{attr_prefix}_bounds", latest.get('new_bounds')) + else: + setattr(fit, f"{attr_prefix}_edge", None) + setattr(fit, f"{attr_prefix}_mode", None) + setattr(fit, f"{attr_prefix}_std", None) + setattr(fit, f"{attr_prefix}_original_bounds", None) + setattr(fit, f"{attr_prefix}_bounds", None) + + +def annotate_sparse_posterior_live_point_extension( + fit, + enabled, + applied, + note=None, + diagnostics=None, + post_extension_diagnostics=None, + base_live_points=None, + target_live_points=None, + extension_factor=SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT, +): + if fit is None: + return + + fit.sparse_posterior_live_point_extension_enabled = bool(enabled) + fit.sparse_posterior_live_point_extension_applied = bool(applied) + fit.sparse_posterior_live_point_extension_note = note + fit.sparse_posterior_live_point_extension_diagnostics = diagnostics + fit.sparse_posterior_live_point_extension_post_diagnostics = post_extension_diagnostics + fit.sparse_posterior_live_point_extension_base_live_points = base_live_points + fit.sparse_posterior_live_point_extension_target_live_points = target_live_points + fit.sparse_posterior_live_point_extension_factor = extension_factor + + +def clear_fit_ultranest_resume_state(fit): + clear_resume_state = getattr(fit, 'clear_ultranest_resume_state', None) + if callable(clear_resume_state): + clear_resume_state() + + +def callable_accepts_keyword(callable_obj, keyword): + try: + signature = inspect.signature(callable_obj) + except (TypeError, ValueError): + return False + + if keyword in signature.parameters: + return True + return any( + parameter.kind == inspect.Parameter.VAR_KEYWORD + for parameter in signature.parameters.values() + ) + + +def add_exposure_times_to_lc_fitter_kwargs(fit_kwargs, exposure_times_seconds): + if exposure_times_seconds is None or not callable_accepts_keyword(lc_fitter, 'exposure_times_seconds'): + return fit_kwargs + try: + exposure_times = np.asarray(exposure_times_seconds, dtype=float) + except (TypeError, ValueError): + return fit_kwargs + fit_kwargs['exposure_times_seconds'] = exposure_times + return fit_kwargs + + +def get_configured_ultranest_min_num_live_points(): + return parse_ultranest_min_num_live_points( + os.environ.get( + ULTRANEST_MIN_NUM_LIVE_POINTS_ENV, + ULTRANEST_MIN_NUM_LIVE_POINTS_DEFAULT, + ) + ) + + +def _effective_sample_count(weights, fallback_count): + if weights is None: + return float(fallback_count) + + weights = np.asarray(weights, dtype=float) + finite_weights = weights[np.isfinite(weights) & (weights > 0)] + if finite_weights.size == 0: + return float(fallback_count) + + weight_sum = float(np.sum(finite_weights)) + weight_square_sum = float(np.sum(finite_weights ** 2)) + if not np.isfinite(weight_sum) or not np.isfinite(weight_square_sum) or weight_square_sum <= 0: + return float(fallback_count) + return float((weight_sum ** 2) / weight_square_sum) + + +def _fit_posterior_sample_matrix(fit, parameter_keys): + parameter_keys = list(parameter_keys) + if fit is None or not parameter_keys: + return np.empty((0, 0), dtype=float), None + + sample_points = None + sample_weights = None + try: + sample_points, _, sample_weights = fit._get_triangle_plot_samples() + except Exception: + sample_points = None + + if sample_points is not None: + sample_points = np.asarray(sample_points, dtype=float) + if sample_points.ndim == 2 and sample_points.shape[0] > 0: + sampled_keys = list(getattr(fit, 'sampled_keys', [])) + bounds = getattr(fit, 'bounds', {}) + bound_keys = list(bounds.keys()) if isinstance(bounds, dict) else [] + physical_getter = getattr(fit, '_physical_values_from_sample_point', None) + columns = [] + for key in parameter_keys: + if key in sampled_keys: + key_index = sampled_keys.index(key) + if key_index >= sample_points.shape[1]: + return np.empty((0, len(parameter_keys)), dtype=float), None + columns.append(np.asarray(sample_points[:, key_index], dtype=float)) + elif callable(physical_getter) and bound_keys: + columns.append(np.asarray([ + physical_getter(point, bound_keys, sampled_keys).get(key, np.nan) + for point in sample_points + ], dtype=float)) + else: + break + else: + weights = None + if sample_weights is not None: + sample_weights = np.asarray(sample_weights, dtype=float) + if sample_weights.ndim == 1 and sample_weights.shape[0] == sample_points.shape[0]: + weights = sample_weights + return np.column_stack(columns), weights + + sample_getter = getattr(fit, 'get_parameter_posterior_samples', None) + if not callable(sample_getter): + return np.empty((0, len(parameter_keys)), dtype=float), None + + columns = [] + min_size = None + for key in parameter_keys: + values = np.asarray(sample_getter(key), dtype=float).reshape(-1) + columns.append(values) + min_size = values.size if min_size is None else min(min_size, values.size) + + if min_size is None or min_size == 0: + return np.empty((0, len(parameter_keys)), dtype=float), None + + return np.column_stack([values[:min_size] for values in columns]), None + + +def evaluate_sparse_posterior_sample_support( + fit, + parameter_keys=SPARSE_POSTERIOR_RETRY_PARAMETER_KEYS, + base_live_points=None, + minimum_effective_samples=None, + minimum_occupied_bins=SPARSE_POSTERIOR_MIN_OCCUPIED_BINS, + minimum_occupied_bin_fraction=SPARSE_POSTERIOR_MIN_OCCUPIED_BIN_FRACTION, + minimum_effective_samples_per_occupied_bin=SPARSE_POSTERIOR_MIN_EFFECTIVE_SAMPLES_PER_OCCUPIED_BIN, +): + if base_live_points is None: + base_live_points = get_configured_ultranest_min_num_live_points() + + if minimum_effective_samples is None: + minimum_effective_samples = max( + SPARSE_POSTERIOR_MIN_EFFECTIVE_SAMPLES_FLOOR, + int(np.ceil(SPARSE_POSTERIOR_MIN_EFFECTIVE_SAMPLES_PER_LIVE_POINT * float(base_live_points))), + ) + minimum_effective_samples = int(max(1, minimum_effective_samples)) + minimum_occupied_bins = int(max(1, minimum_occupied_bins)) + minimum_occupied_bin_fraction = float(np.clip(minimum_occupied_bin_fraction, 0.0, 1.0)) + minimum_effective_samples_per_occupied_bin = float( + max(0.0, minimum_effective_samples_per_occupied_bin) + ) + + sample_matrix, sample_weights = _fit_posterior_sample_matrix(fit, parameter_keys) + diagnostics = { + 'sparse': False, + 'reason': None, + 'parameter_keys': list(parameter_keys), + 'base_live_points': int(base_live_points), + 'minimum_effective_samples': minimum_effective_samples, + 'minimum_occupied_bins': minimum_occupied_bins, + 'minimum_occupied_bin_fraction': minimum_occupied_bin_fraction, + 'minimum_effective_samples_per_occupied_bin': minimum_effective_samples_per_occupied_bin, + 'parameters': {}, + } + + if sample_matrix.size == 0 or sample_matrix.shape[0] == 0: + diagnostics['sparse'] = True + diagnostics['reason'] = "posterior samples are unavailable for Rp/R*, Tmid, and a/Rs." + return diagnostics + + sparse_reasons = [] + for column_index, key in enumerate(parameter_keys): + if column_index >= sample_matrix.shape[1]: + sample_values = np.array([], dtype=float) + else: + sample_values = np.asarray(sample_matrix[:, column_index], dtype=float) + finite_mask = np.isfinite(sample_values) + finite_values = sample_values[finite_mask] + parameter_weights = sample_weights[finite_mask] if sample_weights is not None else None + sample_count = int(finite_values.size) + effective_count = _effective_sample_count(parameter_weights, sample_count) + + occupied_bins = 0 + central_count = 0 + bin_count = 0 + occupied_bin_fraction = 0.0 + effective_samples_per_occupied_bin = 0.0 + if sample_count >= 2: + q05, q95 = np.nanpercentile(finite_values, [5, 95]) + central_mask = (finite_values >= q05) & (finite_values <= q95) + central_values = finite_values[central_mask] + central_count = int(central_values.size) + if np.isfinite(q05) and np.isfinite(q95) and q05 < q95 and central_count > 0: + bin_count = int(np.clip(np.sqrt(sample_count), 10, 40)) + hist_counts, _ = np.histogram(central_values, bins=bin_count, range=(q05, q95)) + occupied_bins = int(np.count_nonzero(hist_counts > 0)) + occupied_bin_fraction = ( + float(occupied_bins) / float(bin_count) + if bin_count > 0 + else 0.0 + ) + if occupied_bins > 0: + effective_samples_per_occupied_bin = float(effective_count) / float(occupied_bins) + + parameter_diagnostic = { + 'sample_count': sample_count, + 'effective_sample_count': float(effective_count), + 'central_sample_count': central_count, + 'central_bin_count': bin_count, + 'occupied_bins': occupied_bins, + 'occupied_bin_fraction': occupied_bin_fraction, + 'effective_samples_per_occupied_bin': effective_samples_per_occupied_bin, + 'sparse': False, + 'reason': None, + } + + if effective_count < minimum_effective_samples: + parameter_diagnostic['sparse'] = True + parameter_diagnostic['reason'] = ( + f"effective samples {effective_count:.0f} < {minimum_effective_samples}" + ) + elif occupied_bins and occupied_bins < minimum_occupied_bins: + parameter_diagnostic['sparse'] = True + parameter_diagnostic['reason'] = ( + f"central posterior occupies {occupied_bins} histogram bins < {minimum_occupied_bins}" + ) + elif bin_count and occupied_bin_fraction < minimum_occupied_bin_fraction: + parameter_diagnostic['sparse'] = True + parameter_diagnostic['reason'] = ( + f"central posterior occupies {occupied_bin_fraction:.2f} of histogram bins " + f"< {minimum_occupied_bin_fraction:.2f}" + ) + elif ( + occupied_bins + and minimum_effective_samples_per_occupied_bin > 0 + and effective_samples_per_occupied_bin < minimum_effective_samples_per_occupied_bin + ): + parameter_diagnostic['sparse'] = True + parameter_diagnostic['reason'] = ( + f"effective samples per occupied bin {effective_samples_per_occupied_bin:.1f} " + f"< {minimum_effective_samples_per_occupied_bin:.1f}" + ) + + if parameter_diagnostic['sparse']: + sparse_reasons.append(f"{key}: {parameter_diagnostic['reason']}") + diagnostics['parameters'][key] = parameter_diagnostic + + if sparse_reasons: + diagnostics['sparse'] = True + diagnostics['reason'] = "; ".join(sparse_reasons) + else: + diagnostics['reason'] = "posterior sample support is sufficient for Rp/R*, Tmid, and a/Rs." + + return diagnostics + + +def sparse_posterior_diagnostics_summary(diagnostics): + if not isinstance(diagnostics, dict): + return "posterior sample support diagnostics are unavailable" + reason = diagnostics.get('reason') + if reason: + return str(reason) + return "posterior sample support diagnostics are unavailable" + + +def extend_sparse_posterior_live_points_if_needed( + fit, + enabled=None, + extension_factor=SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT, + require_sparse=True, + extension_label="sparse-posterior", +): + if enabled is None: + enabled = should_use_sparse_posterior_live_point_retry( + os.environ.get( + SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_ENV, + SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_DEFAULT, + ) + ) + + if not enabled: + annotate_sparse_posterior_live_point_extension(fit, False, False) + clear_fit_ultranest_resume_state(fit) + return fit + + base_live_points = get_configured_ultranest_min_num_live_points() + diagnostics = evaluate_sparse_posterior_sample_support(fit, base_live_points=base_live_points) + if not diagnostics.get('sparse'): + if require_sparse: + annotate_sparse_posterior_live_point_extension( + fit, + True, + False, + note=f"Not needed; {sparse_posterior_diagnostics_summary(diagnostics)}", + diagnostics=diagnostics, + base_live_points=base_live_points, + extension_factor=extension_factor, + ) + clear_fit_ultranest_resume_state(fit) + return fit + + extender = getattr(fit, 'extend_ultranest_fit', None) + if not callable(extender): + note = ( + "Skipped; the retained UltraNest sampler state is unavailable for an additive " + f"{extension_label} live-point extension." + ) + log_info(f"Warning: {note}", warn=True) + annotate_sparse_posterior_live_point_extension( + fit, + True, + False, + note=note, + diagnostics=diagnostics, + base_live_points=base_live_points, + extension_factor=extension_factor, + ) + clear_fit_ultranest_resume_state(fit) + return fit + + extension_factor = int(max(1, extension_factor)) + target_live_points = int(max( + base_live_points + extension_factor * base_live_points, + base_live_points + 1, + )) + try: + current_max_ncalls = int(float(getattr(fit, 'max_ncalls', 2e5))) + except (TypeError, ValueError): + current_max_ncalls = int(2e5) + target_max_ncalls = int(max(current_max_ncalls, current_max_ncalls * (extension_factor + 1))) + if diagnostics.get('sparse'): + log_info( + "Posterior samples for Rp/R*, Tmid, and a/Rs are sparse " + f"({sparse_posterior_diagnostics_summary(diagnostics)}); continuing UltraNest " + f"from {base_live_points} to {target_live_points} minimum live points " + "using the retained final-pass sampler bounds." + ) + else: + log_info( + f"Continuing the {extension_label} UltraNest fit from {base_live_points} " + f"to {target_live_points} minimum live points using the retained final-pass " + f"sampler bounds ({sparse_posterior_diagnostics_summary(diagnostics)})." + ) + applied = bool(extender(min_num_live_points=target_live_points, max_ncalls=target_max_ncalls)) + post_diagnostics = evaluate_sparse_posterior_sample_support(fit, base_live_points=base_live_points) + + if applied and post_diagnostics.get('sparse'): + note = ( + f"Applied additive {extension_label} UltraNest extension " + f"({base_live_points}->{target_live_points} minimum live points), but " + f"{sparse_posterior_diagnostics_summary(post_diagnostics)}" + ) + log_info( + "Warning: sparse posterior support remains after the additive UltraNest extension; " + "please inspect the triangle plot carefully.", + warn=True, + ) + elif applied: + note = ( + f"Applied additive {extension_label} UltraNest extension " + f"({base_live_points}->{target_live_points} minimum live points)." + ) + else: + note = ( + f"Skipped; UltraNest did not continue the additive {extension_label} extension " + "from the retained sampler state." + ) + + annotate_sparse_posterior_live_point_extension( + fit, + True, + applied, + note=note, + diagnostics=diagnostics, + post_extension_diagnostics=post_diagnostics, + base_live_points=base_live_points, + target_live_points=target_live_points, + extension_factor=extension_factor, + ) + clear_fit_ultranest_resume_state(fit) + return fit + + +def extend_selected_comparison_live_points_if_needed(fit, enabled=None): + return extend_sparse_posterior_live_points_if_needed( + fit, + enabled=enabled, + extension_factor=SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT, + require_sparse=False, + extension_label="selected comparison-star final", + ) + + +def prior_centered_parameter_bounds( + prior_value, + percentage, + minimum_bound, + maximum_bound=None, +): + try: + center = float(prior_value) + percentage = float(percentage) + except (TypeError, ValueError): + return None + + if not np.isfinite(center) or center <= minimum_bound: + return None + if not np.isfinite(percentage) or percentage < 0: + return None + + fraction = percentage / 100.0 + lower_bound = max(float(minimum_bound), center * (1.0 - fraction)) + upper_bound = center * (1.0 + fraction) + if maximum_bound is not None: + upper_bound = min(float(maximum_bound), upper_bound) + if not np.isfinite(lower_bound) or not np.isfinite(upper_bound) or upper_bound <= lower_bound: + return None + return [float(lower_bound), float(upper_bound)] + + +def target_name_is_toi_or_tic(value): + if value is None: + return False + return re.match(r'^\s*(?:TOI|TIC)(?:\s*[-_]?\s*)\d', str(value), flags=re.IGNORECASE) is not None + + +def is_toi_or_tic_target(target): + if isinstance(target, dict): + return any( + target_name_is_toi_or_tic(target.get(key)) + for key in ('pName', 'sName', 'planet_name', 'host_name') + ) + return target_name_is_toi_or_tic(target) + + +def ars_range_restriction_percentage_for_prior(prior): + percentage = float(ARS_RANGE_RESTRICTION_PERCENTAGE) + if ( + is_toi_or_tic_target(prior) + and np.isclose( + percentage, + ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT, + rtol=0.0, + atol=1e-12, + ) + ): + return float(TOI_TIC_ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT) + return percentage + + +def ars_initial_range_percentage_for_prior( + prior, + sigma_multiplier=INITIAL_ARS_BOUND_SIGMA_MULTIPLIER, + fallback_relative_half_width=INITIAL_ARS_BOUND_FALLBACK_RELATIVE_HALF_WIDTH, +): + base_percentage = ars_range_restriction_percentage_for_prior(prior) + if not isinstance(prior, dict): + return float(base_percentage) + + try: + ars = float(prior.get('ars')) + ars_unc = float(prior.get('ars_unc')) + sigma_multiplier = float(sigma_multiplier) + except (TypeError, ValueError): + ars = np.nan + ars_unc = np.nan + sigma_multiplier = INITIAL_ARS_BOUND_SIGMA_MULTIPLIER + + if np.isfinite(ars) and ars > ARS_SEARCH_BOUND_MIN and np.isfinite(ars_unc) and ars_unc > 0: + uncertainty_percentage = 100.0 * sigma_multiplier * ars_unc / ars + else: + uncertainty_percentage = 100.0 * float(fallback_relative_half_width) + + return float(max(base_percentage, uncertainty_percentage)) + + +def ars_posterior_retry_limit_for_prior(prior, requested_max_retries): + try: + requested_max_retries = int(max(0, requested_max_retries)) + except (TypeError, ValueError): + requested_max_retries = ARS_POSTERIOR_MAX_RETRIES_DEFAULT + + if ( + is_toi_or_tic_target(prior) + and requested_max_retries >= ARS_POSTERIOR_MAX_RETRIES_DEFAULT + ): + return max(requested_max_retries, TOI_TIC_ARS_POSTERIOR_MAX_RETRIES_DEFAULT) + return requested_max_retries + + +def configured_prior_centered_bounds_for_key(key, prior): + if not isinstance(prior, dict): + return None + if key == 'rprs': + if not RPRS_RANGE_RESTRICTION_ENABLED: + return None + return rprs_prior_centered_bounds(prior) + if key == 'ars': + if not ARS_RANGE_RESTRICTION_ENABLED: + return None + return prior_centered_parameter_bounds( + prior.get('ars'), + ars_initial_range_percentage_for_prior(prior), + ARS_SEARCH_BOUND_MIN, + ) + return None + + +def intersect_parameter_bounds(bounds, restriction): + if restriction is None: + return None + try: + lower_bound, upper_bound = [ + float(value) for value in np.asarray(bounds, dtype=float).reshape(-1)[:2] + ] + restrict_lower, restrict_upper = [ + float(value) for value in np.asarray(restriction, dtype=float).reshape(-1)[:2] + ] + except (TypeError, ValueError, IndexError): + return None + + lower_bound = max(lower_bound, restrict_lower) + upper_bound = min(upper_bound, restrict_upper) + if not np.isfinite(lower_bound) or not np.isfinite(upper_bound) or upper_bound <= lower_bound: + return None + return [float(lower_bound), float(upper_bound)] + + +def apply_configured_prior_search_restrictions(bounds, prior, allow_ars_expansion=False): + restricted = widen_rprs_bounds_to_data_uncertainty_window(bounds, prior) + for key in ('rprs', 'ars'): + if key not in restricted: + continue + if key == 'ars' and allow_ars_expansion: + continue + restriction = configured_prior_centered_bounds_for_key(key, prior) + intersection = intersect_parameter_bounds(restricted[key], restriction) + if intersection is not None: + restricted[key] = intersection + return restricted + + +def bounds_are_close(bounds_a, bounds_b, atol=1e-12): + if bounds_a is None or bounds_b is None: + return False + try: + array_a = np.asarray(bounds_a, dtype=float).reshape(-1)[:2] + array_b = np.asarray(bounds_b, dtype=float).reshape(-1)[:2] + except (TypeError, ValueError, IndexError): + return False + return array_a.shape == array_b.shape and bool(np.allclose(array_a, array_b, rtol=0.0, atol=atol)) + + +def build_initial_rprs_bounds( + rprs, + lower_scale=INITIAL_RPRS_BOUND_LOWER_SCALE, + upper_scale=INITIAL_RPRS_BOUND_UPPER_SCALE, + rprs_data_uncertainty=None, +): + try: + rprs = float(rprs) + lower_scale = float(lower_scale) + upper_scale = float(upper_scale) + except (TypeError, ValueError): + return [RPRS_SEARCH_BOUND_MIN, RPRS_SEARCH_BOUND_MAX] + + if not np.isfinite(rprs) or rprs <= 0: + return [RPRS_SEARCH_BOUND_MIN, RPRS_SEARCH_BOUND_MAX] + if rprs >= RPRS_SEARCH_BOUND_MAX: + return [RPRS_SEARCH_BOUND_MIN, RPRS_SEARCH_BOUND_MAX] + + lower_bound = max(RPRS_SEARCH_BOUND_MIN, lower_scale * rprs) + upper_bound = min(RPRS_SEARCH_BOUND_MAX, upper_scale * rprs) + if not np.isfinite(upper_bound) or upper_bound <= lower_bound: + lower_bound = RPRS_SEARCH_BOUND_MIN + upper_bound = RPRS_SEARCH_BOUND_MAX + + bounds = [float(lower_bound), float(upper_bound)] + restriction_prior = {'rprs': rprs} + if rprs_data_uncertainty is not None: + restriction_prior['rprs_data_uncertainty'] = rprs_data_uncertainty + restricted_bounds = intersect_parameter_bounds( + bounds, + configured_prior_centered_bounds_for_key('rprs', restriction_prior), + ) + return restricted_bounds if restricted_bounds is not None else bounds + + +def build_initial_ars_bounds( + ars, + ars_unc=None, + sigma_multiplier=INITIAL_ARS_BOUND_SIGMA_MULTIPLIER, + fallback_relative_half_width=INITIAL_ARS_BOUND_FALLBACK_RELATIVE_HALF_WIDTH, + search_restriction_prior=None, +): + try: + ars = float(ars) + except (TypeError, ValueError): + ars = np.nan + + try: + ars_unc = float(ars_unc) + except (TypeError, ValueError): + ars_unc = np.nan + + if not np.isfinite(ars) or ars <= ARS_SEARCH_BOUND_MIN: + return [float(ARS_SEARCH_BOUND_MIN), float(ARS_SEARCH_BOUND_FALLBACK_MAX)] + + restriction_prior = ( + dict(search_restriction_prior) + if isinstance(search_restriction_prior, dict) + else {} + ) + restriction_prior['ars'] = ars + restriction_prior['ars_unc'] = ars_unc + + if np.isfinite(ars_unc) and ars_unc > 0: + half_width = float(max(ARS_SEARCH_BOUND_MIN, sigma_multiplier * ars_unc)) + else: + half_width = float(max(ARS_SEARCH_BOUND_MIN, fallback_relative_half_width * ars)) + if ARS_RANGE_RESTRICTION_ENABLED: + minimum_initial_half_width = ( + ars * ars_initial_range_percentage_for_prior( + restriction_prior, + sigma_multiplier=sigma_multiplier, + fallback_relative_half_width=fallback_relative_half_width, + ) / 100.0 + ) + half_width = float(max(half_width, minimum_initial_half_width)) + + lower_bound = max(float(ARS_SEARCH_BOUND_MIN), float(ars - half_width)) + upper_bound = float(ars + half_width) + if not np.isfinite(upper_bound) or upper_bound <= lower_bound: + upper_bound = float(lower_bound + max(np.finfo(float).eps, ARS_SEARCH_BOUND_MIN)) + + bounds = [float(lower_bound), float(upper_bound)] + restricted_bounds = intersect_parameter_bounds( + bounds, + configured_prior_centered_bounds_for_key('ars', restriction_prior), + ) + return restricted_bounds if restricted_bounds is not None else bounds + + +def build_initial_transit_bounds( + prior, + tmid_bounds, + ars_unc=None, + inclination_half_width=5.0, + rprs_data_uncertainty=None, + search_restriction_prior=None, +): + lower, upper = [float(value) for value in np.asarray(tmid_bounds, dtype=float).reshape(-1)[:2]] + # Keep ars ahead of inc so the internal impact-parameter parameterization + # uses the sampled ars value when converting inclination to b. + return { + 'rprs': build_initial_rprs_bounds( + prior['rprs'], + rprs_data_uncertainty=rprs_data_uncertainty, + ), + 'tmid': [lower, upper], + 'ars': build_initial_ars_bounds( + prior['ars'], + ars_unc=ars_unc, + search_restriction_prior=search_restriction_prior, + ), + 'inc': [prior['inc'] - inclination_half_width, min(90, prior['inc'] + inclination_half_width)], + } + + +def clone_lightcurve_bounds(bounds): + return { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in bounds.items() + } + + +def annotate_posterior_refit_final_bounds(fit, bounds): + if fit is None: + return + fit.posterior_refit_final_bounds = clone_lightcurve_bounds(bounds) + + +def get_posterior_refit_final_bounds(fit, fallback_bounds): + effective_bounds = clone_lightcurve_bounds(fallback_bounds) + fit_bounds = getattr(fit, 'posterior_refit_final_bounds', None) + if not isinstance(fit_bounds, dict): + fit_bounds = {} + for key in ('rprs', 'ars', 'inc'): + refit_bounds = getattr(fit, f'{key}_posterior_refit_bounds', None) + if refit_bounds is not None: + fit_bounds[key] = refit_bounds + + for key, value in fit_bounds.items(): + if not isinstance(value, (list, tuple, np.ndarray)): + continue + try: + lower_bound, upper_bound = [ + float(bound) for bound in np.asarray(value, dtype=float).reshape(-1)[:2] + ] + except (TypeError, ValueError, IndexError): + continue + if ( + np.isfinite(lower_bound) + and np.isfinite(upper_bound) + and lower_bound < upper_bound + ): + effective_bounds[key] = [lower_bound, upper_bound] + return effective_bounds + + +def sanitize_parameter_search_bounds(bounds, key, minimum_bound, maximum_bound=None, fallback_maximum=None): + sanitized = clone_lightcurve_bounds(bounds) + if key not in sanitized: + return sanitized + + try: + lower_bound, upper_bound = [ + float(value) for value in np.asarray(sanitized[key], dtype=float).reshape(-1)[:2] + ] + except (TypeError, ValueError, IndexError): + sanitized[key] = [ + float(minimum_bound), + float(maximum_bound if maximum_bound is not None else fallback_maximum), + ] + return sanitized + + if not np.isfinite(lower_bound) or not np.isfinite(upper_bound): + sanitized[key] = [ + float(minimum_bound), + float(maximum_bound if maximum_bound is not None else fallback_maximum), + ] + return sanitized + + lower_bound = max(float(minimum_bound), float(lower_bound)) + if maximum_bound is not None: + upper_bound = min(float(maximum_bound), float(upper_bound)) + if lower_bound >= upper_bound: + sanitized[key] = [ + float(minimum_bound), + float(maximum_bound if maximum_bound is not None else fallback_maximum), + ] + else: + sanitized[key] = [lower_bound, float(upper_bound)] + return sanitized + + +def sanitize_rprs_search_bounds(bounds): + return sanitize_parameter_search_bounds( + bounds, + 'rprs', + RPRS_SEARCH_BOUND_MIN, + maximum_bound=RPRS_SEARCH_BOUND_MAX, + fallback_maximum=RPRS_SEARCH_BOUND_MAX, + ) + + +def sanitize_ars_search_bounds(bounds): + return sanitize_parameter_search_bounds( + bounds, + 'ars', + ARS_SEARCH_BOUND_MIN, + fallback_maximum=ARS_SEARCH_BOUND_FALLBACK_MAX, + ) + + +def sanitize_inclination_search_bounds(bounds): + return sanitize_parameter_search_bounds( + bounds, + 'inc', + INCLINATION_SEARCH_BOUND_MIN, + maximum_bound=INCLINATION_SEARCH_BOUND_MAX, + fallback_maximum=INCLINATION_SEARCH_BOUND_MAX, + ) + + +def sanitize_retry_search_bounds(bounds): + return sanitize_inclination_search_bounds(sanitize_ars_search_bounds(sanitize_rprs_search_bounds(bounds))) + + +def impact_parameter_scale_for_retry(values): + try: + ars = float(values['ars']) + except (KeyError, TypeError, ValueError): + return np.nan + + try: + ecc = float(values.get('ecc', 0.0)) + except (TypeError, ValueError): + ecc = 0.0 + + try: + omega = np.deg2rad(float(values.get('omega', 0.0))) + except (TypeError, ValueError): + omega = 0.0 + + denom = 1.0 + ecc * np.sin(omega) + if np.isclose(denom, 0.0): + denom = np.finfo(float).eps + scale = ars * (1.0 - ecc ** 2) / denom + return float(scale) if np.isfinite(scale) and scale > 0 else np.nan + + +def impact_parameter_scale_range_for_retry(prior, bounds): + values = dict(prior) + ars_candidates = [] + if 'ars' in bounds: + try: + ars_candidates.extend( + float(value) for value in np.asarray(bounds['ars'], dtype=float).reshape(-1)[:2] + ) + except (TypeError, ValueError, IndexError): + pass + if 'ars' in values: + try: + ars_candidates.append(float(values['ars'])) + except (TypeError, ValueError): + pass + + scales = [] + for ars_value in ars_candidates: + candidate_values = dict(values) + candidate_values['ars'] = ars_value + scale = impact_parameter_scale_for_retry(candidate_values) + if np.isfinite(scale) and scale > 0: + scales.append(scale) + + if not scales: + scale = impact_parameter_scale_for_retry(values) + if np.isfinite(scale) and scale > 0: + scales.append(scale) + + if not scales: + return np.nan, np.nan + return float(np.nanmin(scales)), float(np.nanmax(scales)) + + +def inclination_from_impact_parameter_for_retry(impact_parameter, scale): + if not np.isfinite(scale) or scale <= 0: + return np.nan + try: + impact_parameter = float(impact_parameter) + except (TypeError, ValueError): + return np.nan + cosi = np.clip(impact_parameter / scale, -1.0, 1.0) + return float(np.rad2deg(np.arccos(cosi))) + + +def impact_parameter_retry_proposed_inclination_bounds(diagnostics, current_prior, current_bounds): + if not diagnostics: + return None + if 'inc' not in current_bounds: + return None + + try: + previous_lower, previous_upper = [ + float(value) for value in np.asarray(current_bounds['inc'], dtype=float).reshape(-1)[:2] + ] + proposed_b_lower, proposed_b_upper = [ + float(value) for value in np.asarray(diagnostics.get('bounds'), dtype=float).reshape(-1)[:2] + ] + except (TypeError, ValueError, IndexError): + return None + + if ( + not np.isfinite(previous_lower) + or not np.isfinite(previous_upper) + or previous_lower >= previous_upper + or not np.isfinite(proposed_b_lower) + or not np.isfinite(proposed_b_upper) + or proposed_b_lower >= proposed_b_upper + ): + return None + + min_scale, max_scale = impact_parameter_scale_range_for_retry(current_prior, current_bounds) + if not np.isfinite(min_scale) or not np.isfinite(max_scale): + return None + + new_lower = previous_lower + new_upper = previous_upper + clipped_edge = diagnostics.get('edge') + + if clipped_edge in ('upper', None): + inc_for_upper_b = inclination_from_impact_parameter_for_retry(proposed_b_upper, max_scale) + if np.isfinite(inc_for_upper_b): + new_lower = min(new_lower, inc_for_upper_b) + + if clipped_edge in ('lower', None): + inc_for_lower_b = inclination_from_impact_parameter_for_retry(max(0.0, proposed_b_lower), min_scale) + if np.isfinite(inc_for_lower_b): + new_upper = max(new_upper, inc_for_lower_b) + + return [float(new_lower), float(new_upper)] + + +def clamp_parameter_prior_to_bounds(prior, bounds, key): + clamped = dict(prior) + if key not in clamped or key not in bounds: + return clamped + + try: + parameter_value = float(clamped[key]) + lower_bound, upper_bound = [ + float(value) for value in np.asarray(bounds[key], dtype=float).reshape(-1)[:2] + ] + except (TypeError, ValueError, IndexError): + return clamped + + if ( + np.isfinite(parameter_value) and np.isfinite(lower_bound) and np.isfinite(upper_bound) + and lower_bound < upper_bound + ): + clamped[key] = float(np.clip(parameter_value, lower_bound, upper_bound)) + return clamped + + +def clamp_rprs_prior_to_bounds(prior, bounds): + return clamp_parameter_prior_to_bounds(prior, bounds, 'rprs') + + +def clamp_ars_prior_to_bounds(prior, bounds): + return clamp_parameter_prior_to_bounds(prior, bounds, 'ars') + + +def clamp_inclination_prior_to_bounds(prior, bounds): + return clamp_parameter_prior_to_bounds(prior, bounds, 'inc') + + +def clamp_retry_priors_to_bounds(prior, bounds): + return clamp_inclination_prior_to_bounds( + clamp_ars_prior_to_bounds(clamp_rprs_prior_to_bounds(prior, bounds), bounds), + bounds, + ) + + +def enforce_minimum_parameter_retry_half_width( + mode, + bounds, + min_half_width, + minimum_bound, + maximum_bound=None, +): + try: + lower_bound, upper_bound = [ + float(value) for value in np.asarray(bounds, dtype=float).reshape(-1)[:2] + ] + except (TypeError, ValueError, IndexError): + return bounds + + if not np.isfinite(lower_bound) or not np.isfinite(upper_bound) or lower_bound >= upper_bound: + return bounds + + center = float(mode) if np.isfinite(mode) else float(0.5 * (lower_bound + upper_bound)) + half_width = max(float(min_half_width), 0.0) + expanded_lower = min(lower_bound, center - half_width) + expanded_upper = max(upper_bound, center + half_width) + + if expanded_lower < minimum_bound: + if maximum_bound is None: + expanded_upper = expanded_upper + (minimum_bound - expanded_lower) + else: + expanded_upper = min( + maximum_bound, + expanded_upper + (minimum_bound - expanded_lower), + ) + expanded_lower = minimum_bound + if maximum_bound is not None and expanded_upper > maximum_bound: + expanded_lower = max( + minimum_bound, + expanded_lower - (expanded_upper - maximum_bound), + ) + expanded_upper = maximum_bound + + return [float(expanded_lower), float(expanded_upper)] + + +def enforce_minimum_rprs_retry_half_width(mode, bounds, min_half_width=RPRS_RETRY_MIN_HALF_WIDTH): + return enforce_minimum_parameter_retry_half_width( + mode, + bounds, + min_half_width, + RPRS_SEARCH_BOUND_MIN, + maximum_bound=RPRS_SEARCH_BOUND_MAX, + ) + + +def enforce_minimum_ars_retry_half_width(mode, bounds, min_half_width=ARS_RETRY_MIN_HALF_WIDTH): + return enforce_minimum_parameter_retry_half_width( + mode, + bounds, + min_half_width, + ARS_SEARCH_BOUND_MIN, + ) + + +def run_nested_lightcurve_fit_with_rprs_posterior_retry( + times, + flux_values, + flux_errors, + airmass, + prior, + bounds, + jd_times=None, + exposure_times_seconds=None, + use_impactparameter_rather_than_inclination_to_fit=True, + max_rprs_retries=RPRS_POSTERIOR_MAX_RETRIES_DEFAULT, + duration_prior=None, + max_ars_retries=ARS_POSTERIOR_MAX_RETRIES_DEFAULT, + max_impact_parameter_retries=IMPACT_PARAMETER_POSTERIOR_MAX_RETRIES_DEFAULT, + keep_ultranest_sampler=False, + baseline_fit_mask=None, + fixed_parameter_errors=None, + fixed_flux_baseline=False, + ultranest_min_num_live_points=None, + pre_ultranest_coverage_assessment=None, + search_restriction_prior=None, + use_prior_rprs_when_posterior_pinned=None, +): + if use_prior_rprs_when_posterior_pinned is None: + use_prior_rprs_when_posterior_pinned = RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR + else: + use_prior_rprs_when_posterior_pinned = bool(use_prior_rprs_when_posterior_pinned) + restriction_reference_prior = ( + dict(search_restriction_prior) + if isinstance(search_restriction_prior, dict) + else dict(prior) if isinstance(prior, dict) else {} + ) + + def impact_parameter_retry_available(fit, local_bounds): + if not use_impactparameter_rather_than_inclination_to_fit or 'inc' not in local_bounds: + return False + if getattr(fit, 'impact_parameter_sampled_directly', False): + return False + sampled_keys = getattr(fit, 'sampled_keys', []) or [] + sample_bounds = getattr(fit, 'sample_bounds', {}) + return 'b' in sampled_keys or (isinstance(sample_bounds, dict) and 'b' in sample_bounds) + + def identity_retry_bounds(diagnostics, local_prior, local_bounds, config): + return diagnostics.get('bounds') if diagnostics else None + + def impact_parameter_retry_bounds(diagnostics, local_prior, local_bounds, config): + return impact_parameter_retry_proposed_inclination_bounds( + diagnostics, + local_prior, + local_bounds, + ) + + def normal_retry_expands(previous_bounds, new_bounds, clipped_edge, config): + previous_lower, previous_upper = [ + float(value) for value in np.asarray(previous_bounds, dtype=float).reshape(-1)[:2] + ] + new_lower, new_upper = [ + float(value) for value in np.asarray(new_bounds, dtype=float).reshape(-1)[:2] + ] + if clipped_edge == 'upper': + return new_upper > previous_upper + 1e-12 + if clipped_edge == 'lower': + return new_lower < previous_lower - 1e-12 + return new_lower < previous_lower - 1e-12 or new_upper > previous_upper + 1e-12 + + def impact_parameter_retry_expands(previous_bounds, new_bounds, clipped_edge, config): + previous_lower, previous_upper = [ + float(value) for value in np.asarray(previous_bounds, dtype=float).reshape(-1)[:2] + ] + new_lower, new_upper = [ + float(value) for value in np.asarray(new_bounds, dtype=float).reshape(-1)[:2] + ] + if clipped_edge == 'upper': + return new_lower < previous_lower - 1e-12 + if clipped_edge == 'lower': + return new_upper > previous_upper + 1e-12 + return new_lower < previous_lower - 1e-12 or new_upper > previous_upper + 1e-12 + + partial_retry_limits = partial_transit_geometry_retry_limits(pre_ultranest_coverage_assessment) + effective_max_ars_retries = ars_posterior_retry_limit_for_prior( + restriction_reference_prior, + max_ars_retries, + ) + retry_configs = [ + { + 'key': 'rprs', + 'diagnostic_key': 'rprs', + 'bounds_key': 'rprs', + 'label': 'Rp/R*', + 'sanitize_bounds': sanitize_rprs_search_bounds, + 'enforce_half_width': enforce_minimum_rprs_retry_half_width, + 'propose_bounds': identity_retry_bounds, + 'expands_bounds': normal_retry_expands, + 'max_retries': min( + max_rprs_retries, + partial_retry_limits['max_retries']['rprs'], + ) if partial_retry_limits['active'] else max_rprs_retries, + 'requested_max_retries': max_rprs_retries, + 'min_bound': RPRS_SEARCH_BOUND_MIN, + 'max_bound': RPRS_SEARCH_BOUND_MAX, + 'prior_mode_key': 'rprs', + 'annotate': annotate_rprs_posterior_refit, + }, + { + 'key': 'ars', + 'diagnostic_key': 'ars', + 'bounds_key': 'ars', + 'label': 'a/Rs', + 'sanitize_bounds': sanitize_ars_search_bounds, + 'enforce_half_width': enforce_minimum_ars_retry_half_width, + 'propose_bounds': identity_retry_bounds, + 'expands_bounds': normal_retry_expands, + 'max_retries': min( + effective_max_ars_retries, + partial_retry_limits['max_retries']['ars'], + ) if partial_retry_limits['active'] else effective_max_ars_retries, + 'requested_max_retries': effective_max_ars_retries, + 'min_bound': ARS_SEARCH_BOUND_MIN, + 'max_bound': None, + 'prior_mode_key': 'ars', + 'annotate': annotate_ars_posterior_refit, + }, + { + 'key': 'b', + 'diagnostic_key': 'b', + 'bounds_key': 'inc', + 'label': 'impact parameter', + 'sanitize_bounds': sanitize_inclination_search_bounds, + 'enforce_half_width': lambda mode, bounds: bounds, + 'propose_bounds': impact_parameter_retry_bounds, + 'expands_bounds': impact_parameter_retry_expands, + 'max_retries': min( + max_impact_parameter_retries, + partial_retry_limits['max_retries']['b'], + ) if partial_retry_limits['active'] else max_impact_parameter_retries, + 'requested_max_retries': max_impact_parameter_retries, + 'min_bound': INCLINATION_SEARCH_BOUND_MIN, + 'max_bound': INCLINATION_SEARCH_BOUND_MAX, + 'prior_mode_key': None, + 'available': impact_parameter_retry_available, + 'annotate': annotate_impact_parameter_posterior_refit, + }, + ] + base_fixed_parameter_errors = ( + dict(fixed_parameter_errors) + if isinstance(fixed_parameter_errors, dict) + else {} + ) + + def build_fit( + local_prior, + local_bounds, + fixed_parameter_errors_override=None, + allow_ars_expansion=False, + ultranest_warmstart_source=None, + ): + local_bounds = sanitize_retry_search_bounds( + apply_configured_prior_search_restrictions( + local_bounds, + restriction_reference_prior, + allow_ars_expansion=allow_ars_expansion, + ) + ) + effective_fixed_parameter_errors = ( + dict(base_fixed_parameter_errors) + if isinstance(base_fixed_parameter_errors, dict) + else {} + ) + if isinstance(fixed_parameter_errors_override, dict): + effective_fixed_parameter_errors.update(fixed_parameter_errors_override) + local_prior, local_bounds, effective_fixed_parameter_errors, prior_assumption = ( + apply_partial_transit_geometry_prior_assumption( + local_prior, + local_bounds, + pre_ultranest_coverage_assessment, + flux_values=flux_values, + flux_errors=flux_errors, + airmass=airmass, + fixed_parameter_errors=effective_fixed_parameter_errors, + search_restriction_prior=restriction_reference_prior, + ) + ) + effective_fixed_flux_baseline = ( + fixed_flux_baseline + and prior_assumption.get('mode') != 'tmid_baseline_airmass' + ) + if effective_fixed_flux_baseline: + local_bounds = clone_lightcurve_bounds(local_bounds) + for key in ('a0', 'a1', 'a2'): + local_bounds.pop(key, None) + local_prior = clamp_retry_priors_to_bounds(local_prior, local_bounds) + fit_kwargs = { + 'jd_times': jd_times, + 'mode': 'ns', + 'use_impactparameter_rather_than_inclination_to_fit': + use_impactparameter_rather_than_inclination_to_fit, + } + add_exposure_times_to_lc_fitter_kwargs(fit_kwargs, exposure_times_seconds) + if isinstance(duration_prior, dict) and duration_prior.get('applied'): + fit_kwargs['duration_prior'] = duration_prior + if keep_ultranest_sampler and callable_accepts_keyword(lc_fitter, 'keep_ultranest_sampler'): + fit_kwargs['keep_ultranest_sampler'] = True + if baseline_fit_mask is not None and callable_accepts_keyword(lc_fitter, 'baseline_fit_mask'): + fit_kwargs['baseline_fit_mask'] = baseline_fit_mask + if effective_fixed_parameter_errors and callable_accepts_keyword(lc_fitter, 'fixed_parameter_errors'): + fit_kwargs['fixed_parameter_errors'] = effective_fixed_parameter_errors + if effective_fixed_flux_baseline and callable_accepts_keyword(lc_fitter, 'fixed_flux_baseline'): + fit_kwargs['fixed_flux_baseline'] = True + if ( + ultranest_min_num_live_points is not None + and callable_accepts_keyword(lc_fitter, 'ultranest_min_num_live_points') + ): + fit_kwargs['ultranest_min_num_live_points'] = ultranest_min_num_live_points + if ( + ultranest_warmstart_source is not None + and callable_accepts_keyword(lc_fitter, 'ultranest_warmstart_source') + ): + fit_kwargs['ultranest_warmstart_source'] = ultranest_warmstart_source + fit = lc_fitter( + times, + flux_values, + flux_errors, + airmass, + local_prior, + local_bounds, + **fit_kwargs, + ) + annotate_duration_prior(fit, duration_prior) + annotate_partial_transit_geometry_prior_assumption(fit, prior_assumption) + return fit + + current_bounds = sanitize_retry_search_bounds( + apply_configured_prior_search_restrictions(bounds, restriction_reference_prior) + ) + current_prior, current_bounds, base_fixed_parameter_errors, initial_prior_assumption = ( + apply_partial_transit_geometry_prior_assumption( + prior, + current_bounds, + pre_ultranest_coverage_assessment, + flux_values=flux_values, + flux_errors=flux_errors, + airmass=airmass, + fixed_parameter_errors=base_fixed_parameter_errors, + search_restriction_prior=restriction_reference_prior, + ) + ) + current_prior = clamp_retry_priors_to_bounds(current_prior, current_bounds) + retry_histories = {config['key']: [] for config in retry_configs} + retry_notes = {config['key']: None for config in retry_configs} + latest_diagnostics = {config['key']: None for config in retry_configs} + blocked_retry_keys = set() + ars_range_expansion_active = False + fit = build_fit(current_prior, current_bounds) + + while True: + diagnostics_getter = getattr(fit, "get_parameter_posterior_recenter_diagnostics", None) + if not callable(diagnostics_getter): + latest_diagnostics = {config['key']: None for config in retry_configs} + break + + retry_config = None + diagnostics = None + for config in retry_configs: + key = config['key'] + diagnostic_key = config.get('diagnostic_key', key) + bounds_key = config.get('bounds_key', key) + if bounds_key not in current_bounds: + continue + + available = config.get('available') + if callable(available) and not available(fit, current_bounds): + continue + + parameter_diagnostics = diagnostics_getter(diagnostic_key) + latest_diagnostics[key] = parameter_diagnostics + if key in blocked_retry_keys: + continue + + new_bounds = parameter_diagnostics.get('bounds') if parameter_diagnostics else None + max_retries_allowed = int(max(0, config['max_retries'])) + if len(retry_histories[key]) >= max_retries_allowed: + if ( + parameter_diagnostics + and parameter_diagnostics.get('clipped') + and retry_notes[key] is None + ): + requested_max_retries = int(max(0, config.get('requested_max_retries', config['max_retries']))) + if requested_max_retries <= 0: + retry_notes[key] = ( + f"Skipped; automatic {config['label']} posterior range refits are disabled " + "for this fit." + ) + elif partial_retry_limits['active']: + retry_notes[key] = partial_retry_limits['note'] + continue + + if parameter_diagnostics and parameter_diagnostics.get('clipped') and new_bounds is not None: + retry_config = config + diagnostics = parameter_diagnostics + break + + if retry_config is None: + break + + key = retry_config['key'] + bounds_key = retry_config.get('bounds_key', key) + label = retry_config['label'] + new_bounds = retry_config.get('propose_bounds', identity_retry_bounds)( + diagnostics, + current_prior, + current_bounds, + retry_config, + ) + try: + new_lower, new_upper = [float(value) for value in new_bounds] + except (TypeError, ValueError): + retry_notes[key] = f"Skipped; the automatic {label} retry proposed malformed bounds." + blocked_retry_keys.add(key) + continue + if not np.isfinite(new_lower) or not np.isfinite(new_upper) or new_lower >= new_upper: + retry_notes[key] = f"Skipped; the automatic {label} retry proposed invalid bounds." + blocked_retry_keys.add(key) + continue + + previous_bounds = current_bounds.get(bounds_key) + allow_ars_expansion = ars_range_expansion_active or key == 'ars' + clamped_bounds = retry_config['sanitize_bounds']({bounds_key: [new_lower, new_upper]}).get( + bounds_key, + [new_lower, new_upper], + ) + clamped_bounds = retry_config['enforce_half_width']( + diagnostics.get('mode', np.nan), + clamped_bounds, + ) + clamped_bounds = retry_config['sanitize_bounds']({bounds_key: clamped_bounds}).get(bounds_key, clamped_bounds) + clamped_bounds = apply_configured_prior_search_restrictions( + {bounds_key: clamped_bounds}, + restriction_reference_prior, + allow_ars_expansion=allow_ars_expansion, + ).get(bounds_key, clamped_bounds) + clamped_bounds = retry_config['sanitize_bounds']({bounds_key: clamped_bounds}).get(bounds_key, clamped_bounds) + if previous_bounds is not None: + previous_lower, previous_upper = [ + float(value) + for value in np.asarray(previous_bounds, dtype=float).reshape(-1)[:2] + ] + clamped_bounds = [ + min(previous_lower, float(clamped_bounds[0])), + max(previous_upper, float(clamped_bounds[1])), + ] + clamped_bounds = retry_config['sanitize_bounds']( + {bounds_key: clamped_bounds} + ).get(bounds_key, clamped_bounds) + clamped_bounds = apply_configured_prior_search_restrictions( + {bounds_key: clamped_bounds}, + restriction_reference_prior, + allow_ars_expansion=allow_ars_expansion, + ).get(bounds_key, clamped_bounds) + clamped_bounds = retry_config['sanitize_bounds']( + {bounds_key: clamped_bounds} + ).get(bounds_key, clamped_bounds) + new_lower, new_upper = [float(value) for value in clamped_bounds] + if previous_bounds is not None: + clipped_edge = diagnostics.get('edge') + preserves_previous_range = ( + new_lower <= previous_lower + 1e-12 + and new_upper >= previous_upper - 1e-12 + ) + if not preserves_previous_range: + retry_notes[key] = ( + f"Skipped; the automatic {label} retry could not preserve the complete " + "previous sampled range while expanding the prior." + ) + blocked_retry_keys.add(key) + continue + expands_sampled_range = retry_config.get('expands_bounds', normal_retry_expands)( + previous_bounds, + [new_lower, new_upper], + clipped_edge, + retry_config, + ) + + if not expands_sampled_range: + maximum_bound = retry_config['max_bound'] + restricted_bounds = ( + None + if bounds_key == 'ars' and allow_ars_expansion + else configured_prior_centered_bounds_for_key( + bounds_key, + restriction_reference_prior, + ) + ) + if ( + restricted_bounds is not None + and bounds_are_close(previous_bounds, restricted_bounds) + ): + retry_notes[key] = ( + f"Skipped; the automatic {label} retry reached the configured " + f"prior-centered search range [{restricted_bounds[0]:.6f}, " + f"{restricted_bounds[1]:.6f}]." + ) + elif ( + maximum_bound is not None and + previous_lower <= retry_config['min_bound'] + 1e-12 and + previous_upper >= maximum_bound - 1e-12 + ): + retry_notes[key] = ( + f"Skipped; the automatic {label} retry reached the maximum exoplanet " + f"search range [{retry_config['min_bound']:.6f}, {maximum_bound:.6f}]." + ) + else: + retry_notes[key] = f"Skipped; the automatic {label} retry did not expand the sampled range." + blocked_retry_keys.add(key) + continue + + retry_histories[key].append({ + 'attempt': len(retry_histories[key]) + 1, + 'edge': diagnostics.get('edge'), + 'mode': float(diagnostics.get('mode', np.nan)), + 'std': float(diagnostics.get('std', np.nan)), + 'original_bounds': None if previous_bounds is None else [float(previous_bounds[0]), float(previous_bounds[1])], + 'new_bounds': [new_lower, new_upper], + }) + log_info( + f"{label} posterior is truncated against the " + f"{diagnostics.get('edge', 'active')} search bound; retrying nested fit " + f"with {label} centered at {diagnostics.get('mode', np.nan):.6f} " + f"and sigma {diagnostics.get('std', np.nan):.6f} " + f"over [{new_lower:.6f}, {new_upper:.6f}]." + ) + + updated_bounds = clone_lightcurve_bounds(current_bounds) + updated_bounds[bounds_key] = [new_lower, new_upper] + updated_bounds = sanitize_retry_search_bounds( + apply_configured_prior_search_restrictions( + updated_bounds, + restriction_reference_prior, + allow_ars_expansion=allow_ars_expansion, + ) + ) + + updated_prior = dict(current_prior) + fit_parameters = getattr(fit, 'parameters', {}) + if isinstance(fit_parameters, dict): + for bound_key in updated_bounds: + if bound_key in fit_parameters: + updated_prior[bound_key] = fit_parameters[bound_key] + prior_mode_key = retry_config.get('prior_mode_key', key) + if prior_mode_key is not None and np.isfinite(diagnostics.get('mode', np.nan)): + updated_prior[prior_mode_key] = float(diagnostics['mode']) + updated_prior = clamp_retry_priors_to_bounds(updated_prior, updated_bounds) + + current_prior = updated_prior + current_bounds = updated_bounds + ars_range_expansion_active = allow_ars_expansion + previous_fit = fit + fit = build_fit( + current_prior, + current_bounds, + allow_ars_expansion=ars_range_expansion_active, + ultranest_warmstart_source=previous_fit, + ) + warmstart_note = getattr( + fit, + 'ultranest_expanded_prior_warmstart_note', + None, + ) + if getattr(fit, 'ultranest_expanded_prior_warmstart_applied', False): + log_info(warmstart_note) + elif ( + getattr(fit, 'ultranest_expanded_prior_warmstart_attempted', False) + and warmstart_note + ): + log_info( + f"Warning: expanded-prior UltraNest warm start was not used. {warmstart_note}", + warn=True, + ) + + final_diagnostics_getter = getattr(fit, "get_parameter_posterior_recenter_diagnostics", None) + rprs_final_diagnostics = None + if callable(final_diagnostics_getter) and 'rprs' in current_bounds: + rprs_final_diagnostics = final_diagnostics_getter('rprs') + latest_diagnostics['rprs'] = rprs_final_diagnostics + elif latest_diagnostics.get('rprs') is not None: + rprs_final_diagnostics = latest_diagnostics['rprs'] + + if ( + use_prior_rprs_when_posterior_pinned + and 'rprs' in current_bounds + and isinstance(rprs_final_diagnostics, dict) + and rprs_final_diagnostics.get('clipped') + ): + prior_rprs = restriction_reference_prior.get('rprs', prior.get('rprs') if isinstance(prior, dict) else np.nan) + try: + prior_rprs = float(prior_rprs) + except (TypeError, ValueError): + prior_rprs = np.nan + if np.isfinite(prior_rprs) and prior_rprs > 0: + original_fit = fit + original_bounds = clone_lightcurve_bounds(current_bounds) + original_rprs_value = (getattr(original_fit, 'parameters', {}) or {}).get('rprs', np.nan) + fixed_prior = dict(current_prior) + fixed_prior['rprs'] = prior_rprs + fixed_bounds = clone_lightcurve_bounds(current_bounds) + fixed_bounds.pop('rprs', None) + + prefit_uncertainty = estimate_rprs_data_uncertainty_from_lightcurve( + times, + flux_values, + flux_errors, + fixed_prior, + ) + data_rprs_uncertainty = prefit_uncertainty.get( + 'data_rprs_uncertainty', + restriction_reference_prior.get('rprs_data_uncertainty', np.nan), + ) + try: + data_rprs_uncertainty = float(data_rprs_uncertainty) + except (TypeError, ValueError): + data_rprs_uncertainty = np.nan + + fixed_error_override = {} + if np.isfinite(data_rprs_uncertainty) and data_rprs_uncertainty >= 0: + fixed_error_override['rprs'] = data_rprs_uncertainty + + log_info( + "Rp/R* posterior is pinned against the " + f"{rprs_final_diagnostics.get('edge', 'active')} bound while Rp/R* " + "posterior expansion is disabled, exhausted, or blocked; rerunning UltraNest with Rp/R* fixed " + f"to the input prior ({prior_rprs:.6f}) and using a data-only Rp/R* uncertainty." + ) + fallback_fit = build_fit( + fixed_prior, + fixed_bounds, + fixed_parameter_errors_override=fixed_error_override, + allow_ars_expansion=ars_range_expansion_active, + ) + fallback_parameters = getattr(fallback_fit, 'parameters', None) + if isinstance(fallback_parameters, dict): + fallback_parameters['rprs'] = prior_rprs + fallback_errors = getattr(fallback_fit, 'errors', None) + if not isinstance(fallback_errors, dict): + fallback_fit.errors = {} + fallback_errors = fallback_fit.errors + + fallback_fit.rprs_prior_fallback_applied = True + fallback_fit.rprs_prior_fallback_prior_value = prior_rprs + fallback_fit.rprs_prior_fallback_original_fit_value = original_rprs_value + fallback_fit.rprs_prior_fallback_original_bounds = original_bounds.get('rprs') + fallback_fit.rprs_prior_fallback_edge = rprs_final_diagnostics.get('edge') + fallback_fit.rprs_prior_fallback_original_diagnostics = dict(rprs_final_diagnostics) + + empirical_uncertainty = fit_empirical_transit_uncertainty(fallback_fit) + if isinstance(empirical_uncertainty, dict) and empirical_uncertainty.get('available'): + fallback_fit.empirical_transit_uncertainty = empirical_uncertainty + empirical_data_uncertainty = empirical_uncertainty.get('data_rprs_uncertainty') + try: + empirical_data_uncertainty = float(empirical_data_uncertainty) + except (TypeError, ValueError): + empirical_data_uncertainty = np.nan + if np.isfinite(empirical_data_uncertainty) and empirical_data_uncertainty >= 0: + data_rprs_uncertainty = empirical_data_uncertainty + + if np.isfinite(data_rprs_uncertainty) and data_rprs_uncertainty >= 0: + fallback_errors['rprs'] = data_rprs_uncertainty + fixed_parameter_errors_payload = getattr(fallback_fit, 'fixed_parameter_errors', None) + if not isinstance(fixed_parameter_errors_payload, dict): + fallback_fit.fixed_parameter_errors = {} + fixed_parameter_errors_payload = fallback_fit.fixed_parameter_errors + fixed_parameter_errors_payload['rprs'] = data_rprs_uncertainty + + fallback_note = ( + "Applied Rp/R* prior fallback; the sampled Rp/R* posterior hugged the " + f"{rprs_final_diagnostics.get('edge', 'active')} search bound while automatic " + "Rp/R* posterior expansion was disabled, exhausted, or blocked. EXOTIC reran UltraNest with Rp/R* fixed " + f"to the input prior ({prior_rprs:.6f}) and treats Tmid, a/Rs, and " + "impact parameter/inclination as the fitted transit-shape parameters. " + "The quoted Rp/R* uncertainty is a data-only red-noise estimate rather than a " + "model posterior uncertainty." + ) + if np.isfinite(data_rprs_uncertainty) and data_rprs_uncertainty >= 0: + fallback_note += f" Data-only Rp/R* uncertainty: {data_rprs_uncertainty:.6f}." + fallback_fit.rprs_prior_fallback_data_uncertainty = data_rprs_uncertainty + fallback_fit.rprs_prior_fallback_note = fallback_note + retry_notes['rprs'] = fallback_note + current_prior = fixed_prior + current_bounds = fixed_bounds + fit = fallback_fit + final_diagnostics_getter = getattr(fit, "get_parameter_posterior_recenter_diagnostics", None) + + ars_final_diagnostics = None + if callable(final_diagnostics_getter) and 'ars' in current_bounds: + ars_final_diagnostics = final_diagnostics_getter('ars') + latest_diagnostics['ars'] = ars_final_diagnostics + elif latest_diagnostics.get('ars') is not None: + ars_final_diagnostics = latest_diagnostics['ars'] + + ars_restriction_bounds = configured_prior_centered_bounds_for_key( + 'ars', + restriction_reference_prior, + ) + ars_pinned_at_prior_restriction = ( + isinstance(ars_final_diagnostics, dict) + and ars_final_diagnostics.get('clipped') + and 'ars' in current_bounds + and ars_restriction_bounds is not None + and bounds_are_close(current_bounds.get('ars'), ars_restriction_bounds) + ) + ars_retries_exhausted = ( + isinstance(ars_final_diagnostics, dict) + and ars_final_diagnostics.get('clipped') + and 'ars' in current_bounds + and len(retry_histories['ars']) >= effective_max_ars_retries + ) + ars_retry_blocked = ( + isinstance(ars_final_diagnostics, dict) + and ars_final_diagnostics.get('clipped') + and 'ars' in current_bounds + and 'ars' in blocked_retry_keys + ) + ars_prior_fallback_required = ( + ars_pinned_at_prior_restriction + or ars_retries_exhausted + or ars_retry_blocked + ) + if ars_prior_fallback_required: + prior_ars = restriction_reference_prior.get( + 'ars', + prior.get('ars') if isinstance(prior, dict) else np.nan, + ) + try: + prior_ars = float(prior_ars) + except (TypeError, ValueError): + prior_ars = np.nan + + if np.isfinite(prior_ars) and prior_ars > ARS_SEARCH_BOUND_MIN: + original_fit = fit + original_bounds = clone_lightcurve_bounds(current_bounds) + original_ars_value = (getattr(original_fit, 'parameters', {}) or {}).get('ars', np.nan) + fixed_prior = dict(current_prior) + fixed_prior['ars'] = prior_ars + fixed_bounds = clone_lightcurve_bounds(current_bounds) + fixed_bounds.pop('ars', None) + + fixed_error_override = {} + existing_errors = getattr(original_fit, 'errors', {}) or {} + if 'rprs' not in fixed_bounds: + fixed_rprs_error = existing_errors.get('rprs', np.nan) + try: + fixed_rprs_error = float(fixed_rprs_error) + except (TypeError, ValueError): + fixed_rprs_error = np.nan + if np.isfinite(fixed_rprs_error) and fixed_rprs_error >= 0: + fixed_error_override['rprs'] = float(fixed_rprs_error) + prior_ars_error = restriction_reference_prior.get('ars_unc', np.nan) + try: + prior_ars_error = float(prior_ars_error) + except (TypeError, ValueError): + prior_ars_error = np.nan + if np.isfinite(prior_ars_error) and prior_ars_error >= 0: + fixed_error_override['ars'] = prior_ars_error + + if ars_retries_exhausted: + fallback_trigger = ( + f"after {len(retry_histories['ars'])} automatic range expansion(s)" + ) + elif ars_retry_blocked: + fallback_trigger = "after the automatic range expansion could not widen the sampled bounds" + else: + fallback_trigger = ( + "at the edge of the initial prior-centered range " + f"[{ars_restriction_bounds[0]:.6f}, {ars_restriction_bounds[1]:.6f}]" + ) + log_info( + "a/Rs posterior remains pinned against the " + f"{ars_final_diagnostics.get('edge', 'active')} edge {fallback_trigger}; " + f"rerunning UltraNest with a/Rs fixed to the input prior ({prior_ars:.6f})." + ) + fallback_fit = build_fit( + fixed_prior, + fixed_bounds, + fixed_parameter_errors_override=fixed_error_override, + ) + fallback_parameters = getattr(fallback_fit, 'parameters', None) + if isinstance(fallback_parameters, dict): + fallback_parameters['ars'] = prior_ars + fallback_errors = getattr(fallback_fit, 'errors', None) + if not isinstance(fallback_errors, dict): + fallback_fit.errors = {} + fallback_errors = fallback_fit.errors + if np.isfinite(prior_ars_error) and prior_ars_error >= 0: + fallback_errors['ars'] = prior_ars_error + fixed_errors = getattr(fallback_fit, 'fixed_parameter_errors', None) + if not isinstance(fixed_errors, dict): + fallback_fit.fixed_parameter_errors = {} + fixed_errors = fallback_fit.fixed_parameter_errors + fixed_errors['ars'] = prior_ars_error + + for attribute_name, attribute_value in getattr(original_fit, '__dict__', {}).items(): + if attribute_name.startswith('rprs_prior_fallback_'): + setattr(fallback_fit, attribute_name, attribute_value) + + fallback_note = ( + "Applied a/Rs prior fallback because the sampled posterior remained pinned against " + f"the {ars_final_diagnostics.get('edge', 'active')} edge {fallback_trigger}. " + "EXOTIC reran UltraNest with a/Rs fixed " + f"to the input prior ({prior_ars:.6f})" + ) + if np.isfinite(prior_ars_error) and prior_ars_error >= 0: + fallback_note += f" with input uncertainty {prior_ars_error:.6f}." + else: + fallback_note += "." + fallback_fit.ars_prior_fallback_applied = True + fallback_fit.ars_prior_fallback_prior_value = prior_ars + fallback_fit.ars_prior_fallback_prior_uncertainty = prior_ars_error + fallback_fit.ars_prior_fallback_original_fit_value = original_ars_value + fallback_fit.ars_prior_fallback_original_bounds = original_bounds.get('ars') + fallback_fit.ars_prior_fallback_edge = ars_final_diagnostics.get('edge') + fallback_fit.ars_prior_fallback_original_diagnostics = dict(ars_final_diagnostics) + fallback_fit.ars_prior_fallback_note = fallback_note + retry_notes['ars'] = fallback_note + current_prior = fixed_prior + current_bounds = fixed_bounds + fit = fallback_fit + final_diagnostics_getter = getattr(fit, "get_parameter_posterior_recenter_diagnostics", None) + + annotate_posterior_refit_final_bounds(fit, current_bounds) + for config in retry_configs: + key = config['key'] + diagnostic_key = config.get('diagnostic_key', key) + bounds_key = config.get('bounds_key', key) + label = config['label'] + history = retry_histories[key] + final_diagnostics = None + available = config.get('available') + config_available = not callable(available) or available(fit, current_bounds) + prior_fallback_applied = bool(getattr(fit, f'{key}_prior_fallback_applied', False)) + if prior_fallback_applied: + final_diagnostics = None + elif callable(final_diagnostics_getter) and bounds_key in current_bounds and config_available: + final_diagnostics = final_diagnostics_getter(diagnostic_key) + elif latest_diagnostics.get(key) is not None: + final_diagnostics = latest_diagnostics[key] + + if prior_fallback_applied: + fallback_note = getattr(fit, f'{key}_prior_fallback_note', None) or retry_notes.get(key) + note = fallback_note + if history: + note = ( + f"Applied {len(history)} automatic {label} posterior range refit(s), " + f"then applied the {label} prior fallback." + ) + elif history: + note = f"Applied {len(history)} automatic {label} posterior range refit(s)." + if final_diagnostics and final_diagnostics.get('clipped'): + retry_label = "retry" if len(history) == 1 else "retries" + note = ( + f"{note} The posterior still hugs the {final_diagnostics.get('edge')} bound after " + f"{len(history)} {retry_label}." + ) + log_info( + f"Warning: {label} posterior still appears truncated after the automatic retries; " + "please inspect the triangle plot carefully.", + warn=True, + ) + elif retry_notes[key] is not None: + note = retry_notes[key] + elif final_diagnostics is not None and final_diagnostics.get('reason'): + note = f"Not needed; {final_diagnostics['reason']}" + else: + note = "Not evaluated; posterior diagnostics are unavailable for this fit." + + config['annotate'](fit, bool(history), note=note, history=history) + return fit + + +def format_clock_log_message(string, clock_time=None): + clock_time = datetime.now() if clock_time is None else clock_time + message = str(string) + leading_newlines = len(message) - len(message.lstrip('\r\n')) + prefix = message[:leading_newlines] + body = message[leading_newlines:] + return f"{prefix}[{clock_time.strftime('%H:%M')}] {body}" + + +def log_info(string, warn=False, error=False): + timestamped_string = format_clock_log_message(string) + if error: + print(f"\033[31m {timestamped_string}\033[0m", flush=True) + elif warn: + print(f"\033[34m {timestamped_string}\033[0m", flush=True) + else: + print(timestamped_string, flush=True) + log.debug(timestamped_string) + _reset_runtime_traceback_watchdog() + return True + + +class ReductionStageTimer: + def __init__(self, time_source=perf_counter): + self._time_source = time_source + self.started_at = float(time_source()) + self.previous_checkpoint = self.started_at + + def checkpoint(self, label): + now = float(self._time_source()) + elapsed_seconds = max(0.0, now - self.previous_checkpoint) + total_seconds = max(0.0, now - self.started_at) + log_info( + f"STEP TIMING | {label} | elapsed_s={elapsed_seconds:.2f} | total_s={total_seconds:.2f}" + ) + self.previous_checkpoint = now + return elapsed_seconds + + +def _find_runtime_handler(handler_name): + for handler in log.handlers: + if getattr(handler, "_exotic_runtime_handler_name", None) == handler_name: + return handler + return None + + +def _runtime_traceback_watchdog_seconds(): + try: + return float(os.environ.get( + _RUNTIME_TRACEBACK_WATCHDOG_SECONDS_ENV, + _RUNTIME_TRACEBACK_WATCHDOG_DEFAULT_SECONDS, + )) + except (TypeError, ValueError): + return _RUNTIME_TRACEBACK_WATCHDOG_DEFAULT_SECONDS + + +def _reset_runtime_traceback_watchdog(): + global _RUNTIME_TRACEBACK_WATCHDOG_ACTIVE + + if not _RUNTIME_LOGGING_CONFIGURED: + return + + timeout = _runtime_traceback_watchdog_seconds() + if timeout <= 0: + cancel_runtime_traceback_watchdog() + return + + try: + faulthandler.cancel_dump_traceback_later() + except Exception: + pass + + try: + faulthandler.dump_traceback_later(timeout, repeat=False, file=sys.stdout) + _RUNTIME_TRACEBACK_WATCHDOG_ACTIVE = True + except Exception: + _RUNTIME_TRACEBACK_WATCHDOG_ACTIVE = False + + +def cancel_runtime_traceback_watchdog(): + global _RUNTIME_TRACEBACK_WATCHDOG_ACTIVE + + if not _RUNTIME_TRACEBACK_WATCHDOG_ACTIVE: + return + + try: + faulthandler.cancel_dump_traceback_later() + except Exception: + pass + _RUNTIME_TRACEBACK_WATCHDOG_ACTIVE = False + + +def _runtime_output_directory_from_command_line(argv=None): + """Return the configured output directory when an init file is on the command line.""" + command_line = list(sys.argv[1:] if argv is None else argv) + init_options = { + '-red', '--reduce', '-pre', '--prereduced', '-phot', '--photometry', '-rt', '--realtime', + } + init_path = None + + for index, argument in enumerate(command_line): + if argument in init_options: + if index + 1 < len(command_line) and command_line[index + 1]: + init_path = command_line[index + 1] + break + for option in init_options: + option_prefix = f"{option}=" + if argument.startswith(option_prefix): + init_path = argument[len(option_prefix):] + break + if init_path is not None: + break + + if not init_path: + return None + + try: + with open(Path(init_path).expanduser(), encoding='utf-8') as init_file: + init_data = json.load(init_file) + output_directory = init_data.get('user_info', {}).get('Directory to Save Plots') + except (OSError, TypeError, ValueError): + return None + + return output_directory or None + + +def _new_runtime_log_basename(): + run_timestamp = datetime.now().strftime("%Y%m%dT%H%M%S_%f") + return f"EXOTIC_RunLog_{run_timestamp}_pid{os.getpid()}.log" + + +def _runtime_log_directory(output_dir=None): + if output_dir: + return Path(output_dir).expanduser().resolve() / "Diagnostics" + return Path(tempfile.gettempdir()).resolve() / "exotic-runtime-logs" + + +def _available_runtime_log_path(directory, basename): + candidate = directory / basename + if not candidate.exists(): + return candidate + + stem = Path(basename).stem + suffix = Path(basename).suffix + duplicate_number = 2 + while True: + candidate = directory / f"{stem}_{duplicate_number}{suffix}" + if not candidate.exists(): + return candidate + duplicate_number += 1 + + +def _runtime_file_formatter(): + return logging.Formatter( + "%(asctime)s.%(msecs)03d [%(threadName)-12.12s] %(levelname)-5.5s " + "%(funcName)s:%(lineno)d - %(message)s", + "%Y-%m-%dT%H:%M:%S", + ) + + +class FailSoftRuntimeFileHandler(logging.FileHandler): + """Keep notebook output usable when a mounted run-log stream disconnects.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._exotic_stream_warning_emitted = False + + def _discard_disconnected_stream(self): + stream = self.stream + self.stream = None + if stream is not None: + try: + stream.close() + except OSError: + pass + + if self._exotic_stream_warning_emitted: + return + self._exotic_stream_warning_emitted = True + try: + print( + "Warning: The EXOTIC run log stream disconnected; console output will continue " + "and EXOTIC will retry the log file automatically.", + file=sys.stdout, + flush=True, + ) + except Exception: + pass + + def emit(self, record): + try: + super().emit(record) + except OSError: + # FileHandler._open() happens outside StreamHandler.emit()'s error + # guard, so mounted-drive failures while reopening need handling here. + self._discard_disconnected_stream() + + def flush(self): + self.acquire() + try: + if self.stream is not None: + try: + self.stream.flush() + except OSError: + self._discard_disconnected_stream() + finally: + self.release() + + def handleError(self, record): + if isinstance(sys.exc_info()[1], OSError): + self._discard_disconnected_stream() + return + super().handleError(record) + + +def _open_runtime_file_handler(log_path): + file_handler = FailSoftRuntimeFileHandler(filename=log_path, mode='a', encoding='utf-8') + file_handler._exotic_runtime_handler_name = _RUNTIME_FILE_HANDLER_NAME + file_handler.setLevel(logging.DEBUG) + file_handler.setFormatter(_runtime_file_formatter()) + log.addHandler(file_handler) + return file_handler + + +def _close_runtime_file_handler(): + file_handler = _find_runtime_handler(_RUNTIME_FILE_HANDLER_NAME) + if file_handler is None: + return + + log.removeHandler(file_handler) + try: + file_handler.flush() + finally: + file_handler.close() + + +def close_runtime_logging(): + global _RUNTIME_LOGGING_CONFIGURED, _RUNTIME_LOG_BASENAME, _RUNTIME_LOG_PATH + + _close_runtime_file_handler() + _RUNTIME_LOGGING_CONFIGURED = False + _RUNTIME_LOG_BASENAME = None + _RUNTIME_LOG_PATH = None + + +def configure_runtime_logging(output_dir=None, start_new_run=False): + global _RUNTIME_LOGGING_CONFIGURED, _RUNTIME_LOG_BASENAME, _RUNTIME_LOG_PATH + + log.setLevel(logging.DEBUG) + # EXOTIC owns both of the handlers it needs below. Do not also propagate + # records into environment-owned root handlers: notebook runtimes such as + # Colab can leave one of those handlers attached to a disconnected output + # transport while the current sys.stdout remains usable. + log.propagate = False + + if start_new_run: + _close_runtime_file_handler() + _RUNTIME_LOG_BASENAME = _new_runtime_log_basename() + _RUNTIME_LOG_PATH = None + if output_dir is None: + output_dir = _runtime_output_directory_from_command_line() + elif _RUNTIME_LOG_BASENAME is None: + _RUNTIME_LOG_BASENAME = _new_runtime_log_basename() + + try: + requested_log_directory = _runtime_log_directory(output_dir) + requested_log_directory.mkdir(parents=True, exist_ok=True) + except Exception as exc: + print(f"Warning: Could not initialize the EXOTIC run log directory ({exc}).") + requested_log_directory = _runtime_log_directory() + requested_log_directory.mkdir(parents=True, exist_ok=True) + output_dir = None + + file_handler = _find_runtime_handler(_RUNTIME_FILE_HANDLER_NAME) + if file_handler is not None and output_dir: + current_log_path = Path(file_handler.baseFilename).resolve() + requested_parent = requested_log_directory.resolve() + if current_log_path.parent != requested_parent: + destination = _available_runtime_log_path(requested_parent, _RUNTIME_LOG_BASENAME) + log.debug(f"Relocating EXOTIC run log to {destination}") + _close_runtime_file_handler() + try: + shutil.move(str(current_log_path), str(destination)) + except Exception as exc: + print(f"Warning: Could not move the EXOTIC run log into Diagnostics ({exc}).") + destination = current_log_path + try: + file_handler = _open_runtime_file_handler(destination) + _RUNTIME_LOG_PATH = destination + _RUNTIME_LOG_BASENAME = destination.name + print(f"EXOTIC run log: {destination}", flush=True) + except Exception as exc: + file_handler = None + print(f"Warning: Could not reopen the EXOTIC run log ({exc}).") + + if file_handler is None: + log_path = _available_runtime_log_path(requested_log_directory, _RUNTIME_LOG_BASENAME) + try: + file_handler = _open_runtime_file_handler(log_path) + except Exception as exc: + print(f"Warning: Could not initialize the EXOTIC run log ({exc}).") + else: + _RUNTIME_LOG_PATH = log_path + _RUNTIME_LOG_BASENAME = log_path.name + print(f"EXOTIC run log: {log_path}", flush=True) + + console_handler = _find_runtime_handler(_RUNTIME_CONSOLE_HANDLER_NAME) + if console_handler is None: + console_handler = logging.StreamHandler(sys.stdout) + console_handler._exotic_runtime_handler_name = _RUNTIME_CONSOLE_HANDLER_NAME + console_handler.setLevel(logging.INFO) + console_handler.setFormatter(logging.Formatter("%(message)s")) + log.addHandler(console_handler) + else: + try: + console_handler.setStream(sys.stdout) + except Exception: + console_handler.stream = sys.stdout + + try: + faulthandler.enable(file=sys.stdout, all_threads=True) + except Exception: + pass + + _RUNTIME_LOGGING_CONFIGURED = True + _reset_runtime_traceback_watchdog() + + +def _logger_has_current_stdout_handler(logger): + current_stdout = sys.stdout + active_logger = logger + while active_logger: + for handler in active_logger.handlers: + if getattr(handler, "stream", None) is current_stdout: + return True + if not getattr(active_logger, "propagate", False): + break + active_logger = active_logger.parent + return False + + +def _write_exception_traceback_to_stdout(message, exc_type, exc_value, exc_traceback): + traceback_text = ''.join(traceback.format_exception(exc_type, exc_value, exc_traceback)) + try: + print(f"\n{message}", file=sys.stdout, flush=True) + print(traceback_text, file=sys.stdout, end="", flush=True) + except Exception: + try: + print(f"\n{message}", file=sys.__stdout__, flush=True) + print(traceback_text, file=sys.__stdout__, end="", flush=True) + except Exception: + pass + + +def _log_exception_with_fallback(message, exc_type, exc_value, exc_traceback): + wrote_to_logger = False + try: + log.error(message, exc_info=(exc_type, exc_value, exc_traceback)) + wrote_to_logger = True + except Exception: + pass + + if not wrote_to_logger or not _logger_has_current_stdout_handler(log): + _write_exception_traceback_to_stdout(message, exc_type, exc_value, exc_traceback) + + +def _handle_unhandled_exception(exc_type, exc_value, exc_traceback): + global _UNHANDLED_EXCEPTION_LOGGED + + if exc_type is not None and issubclass(exc_type, KeyboardInterrupt): + return + + if _UNHANDLED_EXCEPTION_LOGGED: + return + + _UNHANDLED_EXCEPTION_LOGGED = True + _log_exception_with_fallback("Unhandled exception during EXOTIC run", exc_type, exc_value, exc_traceback) + + +def _handle_thread_exception(args): + if args.exc_type is not None and issubclass(args.exc_type, KeyboardInterrupt): + return + + thread_name = args.thread.name if args.thread is not None else "unknown" + _log_exception_with_fallback( + f"Unhandled exception in thread '{thread_name}'", + args.exc_type, + args.exc_value, + args.exc_traceback, + ) + + +def install_exception_hooks(): + global _EXCEPTION_HOOKS_INSTALLED + + if _EXCEPTION_HOOKS_INSTALLED: + return + + sys.excepthook = _handle_unhandled_exception + threading.excepthook = _handle_thread_exception + _EXCEPTION_HOOKS_INSTALLED = True + + +def should_log_plate_solution_path(wcs_file): + if not wcs_file: + return False + + normalized_path = os.fspath(wcs_file).replace("\\", "/") + return normalized_path != "/tmp" and not normalized_path.startswith("/tmp/") + + +def log_mid_transit_range_warning_once(array_times, tmid_prior): + global _mid_transit_warning_reported + if _mid_transit_warning_reported: + return + + _mid_transit_warning_reported = True + # Keep this warning in plain black text and show it once per run. + log_info("\nWarning:") + log_info(" Estimated mid-transit time is not within the observations") + log_info(" Check Period & Mid-transit time in inits.json. Make sure the uncertainties are not 0 or Nan.") + log_info(f" obs start:{array_times.min()}") + log_info(f" obs end:{array_times.max()}") + log_info(f" tmid prior:{tmid_prior}\n") + + +def relative_flux_filter_mask(relative_flux, max_relative_flux=RELATIVE_FLUX_MAX): + relative_flux = np.asarray(relative_flux, dtype=float) + return ( + np.isfinite(relative_flux) + & np.greater(relative_flux, 0) + ) + + +def valid_flux_ratio_mask(relative_flux): + relative_flux = np.asarray(relative_flux, dtype=float) + return np.isfinite(relative_flux) & np.greater(relative_flux, 0) + + +def valid_comparison_frame_mask(flux_values): + flux_values = np.asarray(flux_values, dtype=float) + return np.isfinite(flux_values) & (flux_values > 0) + + +def robust_flux_floor_mask( + flux_values, + min_fraction_of_median=ROBUST_FLUX_MIN_FRACTION_OF_MEDIAN, + min_points=ROBUST_FLUX_MIN_POINTS, +): + flux_values = np.asarray(flux_values, dtype=float) + valid = np.isfinite(flux_values) & (flux_values > 0) + if np.count_nonzero(valid) < max(LIGHTCURVE_MIN_VALID_POINTS, int(min_points)): + return valid + + center, _ = sigma_clipped_nanmedian(flux_values[valid], sigma=4.0, max_iters=3) + if not np.isfinite(center) or center <= 0: + center = bn.nanmedian(flux_values[valid]) + if not np.isfinite(center) or center <= 0: + return valid + + floor = float(min_fraction_of_median) * float(center) + if not np.isfinite(floor) or floor <= 0: + return valid + + return valid & np.greater_equal(flux_values, floor) + + +def robust_target_reference_flux_mask(target_flux, reference_flux): + target_mask = robust_flux_floor_mask(target_flux) + if reference_flux is None: + return target_mask + + reference_mask = robust_flux_floor_mask(reference_flux) + return target_mask & reference_mask + + +def psf_metric_outlier_mask(values, sigma=PSF_FRAME_QUALITY_SIGMA, + max_iters=PSF_FRAME_QUALITY_MAX_CLIP_ITERS, + min_points=LIGHTCURVE_MIN_VALID_POINTS, + high=True, low=True, min_fractional_deviation=0.0): + values = np.asarray(values, dtype=float).reshape(-1) + outlier_mask = ~np.isfinite(values) | (values <= 0) + valid_indices = np.flatnonzero(~outlier_mask) + if valid_indices.size < max(int(min_points), 3): + return outlier_mask + + try: + sigma = float(sigma) + except (TypeError, ValueError): + sigma = PSF_FRAME_QUALITY_SIGMA + if not np.isfinite(sigma) or sigma <= 0: + sigma = PSF_FRAME_QUALITY_SIGMA + + try: + min_fractional_deviation = float(min_fractional_deviation) + except (TypeError, ValueError): + min_fractional_deviation = 0.0 + if not np.isfinite(min_fractional_deviation) or min_fractional_deviation < 0: + min_fractional_deviation = 0.0 + min_log_deviation = np.log1p(min_fractional_deviation) + + log_values = np.log(values[valid_indices]) + keep = np.ones(valid_indices.size, dtype=bool) + max_iters = max(int(max_iters), 1) + min_points = max(int(min_points), 3) + + for _ in range(max_iters): + if np.count_nonzero(keep) < min_points: + break + + kept_values = log_values[keep] + center = bn.nanmedian(kept_values) + if not np.isfinite(center): + break + + scatter = robust_scatter(kept_values - center) + if not np.isfinite(scatter) or scatter <= 0: + break + + deviation = log_values - center + direction_mask = np.zeros(deviation.shape, dtype=bool) + if high: + direction_mask |= deviation > 0 + if low: + direction_mask |= deviation < 0 + + newly_rejected = ( + keep + & direction_mask + & (np.abs(deviation) > sigma * scatter) + & (np.abs(deviation) > min_log_deviation) + ) + if not np.any(newly_rejected): + break + if np.count_nonzero(keep & ~newly_rejected) < min_points: + break + keep[newly_rejected] = False + + outlier_mask[valid_indices] = ~keep + return outlier_mask + + +def psf_frame_quality_components(psf_rows): + psf_rows = np.asarray(psf_rows, dtype=float) + if psf_rows.ndim != 2 or psf_rows.shape[1] < 5: + frame_count = 0 if psf_rows.ndim == 0 else psf_rows.shape[0] + empty = np.zeros(frame_count, dtype=bool) + return { + 'keep_mask': ~empty, + 'invalid_mask': empty, + 'seeing_outlier_mask': empty, + 'amplitude_outlier_mask': empty, + } + + amplitude = psf_rows[:, 2] + sigma_x = psf_rows[:, 3] + sigma_y = psf_rows[:, 4] + seeing = GAUSSIAN_SIGMA_TO_FWHM * 0.5 * (sigma_x + sigma_y) + + invalid_mask = ( + ~np.isfinite(psf_rows[:, 0]) + | ~np.isfinite(psf_rows[:, 1]) + | ~np.isfinite(amplitude) + | ~np.isfinite(sigma_x) + | ~np.isfinite(sigma_y) + | (amplitude <= 0) + | (sigma_x <= 0) + | (sigma_y <= 0) + ) + seeing_outlier_mask = psf_metric_outlier_mask( + seeing, + high=True, + low=False, + min_fractional_deviation=PSF_FRAME_QUALITY_SEEING_MIN_FRACTIONAL_DEVIATION, + ) & ~invalid_mask + amplitude_outlier_mask = psf_metric_outlier_mask( + amplitude, + high=False, + low=True, + min_fractional_deviation=PSF_FRAME_QUALITY_AMPLITUDE_MIN_FRACTIONAL_DEVIATION, + ) & ~invalid_mask + keep_mask = ~(invalid_mask | seeing_outlier_mask | amplitude_outlier_mask) + return { + 'keep_mask': keep_mask, + 'invalid_mask': invalid_mask, + 'seeing_outlier_mask': seeing_outlier_mask, + 'amplitude_outlier_mask': amplitude_outlier_mask, + } + + +def target_psf_shape_quality_components(target_rows, reference_rows=None): + target_rows = np.asarray(target_rows, dtype=float) + if target_rows.ndim != 2 or target_rows.shape[1] < 5: + frame_count = 0 if target_rows.ndim == 0 else target_rows.shape[0] + empty = np.zeros(frame_count, dtype=bool) + return { + 'keep_mask': ~empty, + 'invalid_mask': empty, + 'seeing_outlier_mask': empty, + 'axis_ratio_outlier_mask': empty, + 'reference_width_outlier_mask': empty, + } + + amplitude = target_rows[:, 2] + sigma_x = target_rows[:, 3] + sigma_y = target_rows[:, 4] + seeing = GAUSSIAN_SIGMA_TO_FWHM * 0.5 * (sigma_x + sigma_y) + + invalid_mask = ( + ~np.isfinite(target_rows[:, 0]) + | ~np.isfinite(target_rows[:, 1]) + | ~np.isfinite(amplitude) + | ~np.isfinite(sigma_x) + | ~np.isfinite(sigma_y) + | (amplitude <= 0) + | (sigma_x <= 0) + | (sigma_y <= 0) + ) + seeing_outlier_mask = psf_metric_outlier_mask( + seeing, + high=True, + low=False, + min_fractional_deviation=PSF_FRAME_QUALITY_SEEING_MIN_FRACTIONAL_DEVIATION, + ) & ~invalid_mask + + axis_ratio = np.full(target_rows.shape[0], np.nan, dtype=float) + valid_width = np.isfinite(sigma_x) & np.isfinite(sigma_y) & (sigma_x > 0) & (sigma_y > 0) + axis_ratio[valid_width] = ( + np.maximum(sigma_x[valid_width], sigma_y[valid_width]) + / np.maximum(np.minimum(sigma_x[valid_width], sigma_y[valid_width]), 1e-12) + ) + axis_ratio_outlier_mask = (axis_ratio > PSF_FIT_MAX_AXIS_RATIO) & ~invalid_mask + + reference_width_outlier_mask = np.zeros(target_rows.shape[0], dtype=bool) + if reference_rows is not None: + reference_rows = np.asarray(reference_rows, dtype=float) + if ( + reference_rows.ndim == 2 + and reference_rows.shape[0] == target_rows.shape[0] + and reference_rows.shape[1] >= 5 + ): + reference_sigma_x = reference_rows[:, 3] + reference_sigma_y = reference_rows[:, 4] + reference_sigma = 0.5 * (reference_sigma_x + reference_sigma_y) + target_sigma = 0.5 * (sigma_x + sigma_y) + reference_valid = ( + np.isfinite(reference_sigma) + & np.isfinite(target_sigma) + & (reference_sigma > 0) + & (target_sigma > 0) + ) + reference_width_outlier_mask = ( + reference_valid + & ( + target_sigma + > PSF_TARGET_QUALITY_MAX_COMP_SIGMA_RATIO * reference_sigma + ) + ) + reference_width_outlier_mask &= ~invalid_mask + + keep_mask = ~( + invalid_mask + | seeing_outlier_mask + | axis_ratio_outlier_mask + | reference_width_outlier_mask + ) + return { + 'keep_mask': keep_mask, + 'invalid_mask': invalid_mask, + 'seeing_outlier_mask': seeing_outlier_mask, + 'axis_ratio_outlier_mask': axis_ratio_outlier_mask, + 'reference_width_outlier_mask': reference_width_outlier_mask, + } + + +def target_psf_shape_quality_mask(target_rows, reference_rows=None): + return target_psf_shape_quality_components(target_rows, reference_rows)['keep_mask'] + + +def psf_frame_quality_mask(psf_rows): + return psf_frame_quality_components(psf_rows)['keep_mask'] + + +def psf_quality_rows_for_key(psf_data, key, psf_flux_data=None): + if isinstance(psf_flux_data, dict) and key in psf_flux_data: + return psf_flux_data[key] + if isinstance(psf_data, dict) and key in psf_data: + return psf_data[key] + return None + + +def psf_quality_mask_for_key(psf_data, key, frame_count, psf_flux_data=None): + rows = psf_quality_rows_for_key(psf_data, key, psf_flux_data=psf_flux_data) + if rows is None: + return np.ones(int(frame_count), dtype=bool) + + mask = psf_frame_quality_mask(rows) + if mask.shape[0] != int(frame_count): + return np.ones(int(frame_count), dtype=bool) + return mask + + +def target_psf_quality_rows(psf_data, psf_flux_data=None): + return psf_quality_rows_for_key(psf_data, 'target', psf_flux_data=psf_flux_data) + + +def mask_series_with_quality(values, quality_mask): + masked = np.asarray(values, dtype=float).copy() + quality_mask = np.asarray(quality_mask, dtype=bool) + if masked.shape[0] == quality_mask.shape[0]: + masked[~quality_mask] = np.nan + return masked + + +def psf_flux_series_from_rows(psf_rows, quality_mask=None): + psf_rows = np.asarray(psf_rows, dtype=float) + flux = 2 * np.pi * psf_rows[:, 2] * psf_rows[:, 3] * psf_rows[:, 4] + if quality_mask is not None: + flux = mask_series_with_quality(flux, quality_mask) + return flux + + +def initialize_psf_noise_data(frame_count, comp_star_count): + psf_noise_data = {'target': np.full(int(frame_count), np.nan, dtype=float)} + for component in NOISE_BUDGET_COMPONENT_KEYS: + psf_noise_data[f"target_noise_{component}"] = np.full(int(frame_count), np.nan, dtype=float) + for comp_idx in range(comp_star_count): + ckey = f"comp{comp_idx + 1}" + psf_noise_data[ckey] = np.full(int(frame_count), np.nan, dtype=float) + for component in NOISE_BUDGET_COMPONENT_KEYS: + psf_noise_data[f"{ckey}_noise_{component}"] = np.full(int(frame_count), np.nan, dtype=float) + return psf_noise_data + + +def compute_psf_noise_budget_for_row(data, psf_row, star_index, noise_config=None, + exposure_s=np.nan, airmass=np.nan, fallback_sigma=np.nan, + fast_mode=False): + psf_row = np.asarray(psf_row, dtype=float).reshape(-1) + empty_budget = {component: np.nan for component in NOISE_BUDGET_COMPONENT_KEYS} + if psf_row.shape[0] < 5: + return empty_budget + xc, yc = psf_row[0], psf_row[1] + sigma_x, sigma_y = psf_row[3], psf_row[4] + if not ( + np.isfinite(xc) + and np.isfinite(yc) + and np.isfinite(sigma_x) + and np.isfinite(sigma_y) + and sigma_x > 0 + and sigma_y > 0 + ): + return empty_budget + + flux = psf_flux_series_from_rows(psf_row.reshape(1, -1))[0] + sigma = psf_sigma_from_fit(psf_row, fallback_sigma=fallback_sigma) + if not np.isfinite(sigma) or sigma <= 0: + sigma = max(float(np.sqrt(sigma_x * sigma_y)), 1.0) + effective_pixels = max(PSF_EFFECTIVE_NOISE_AREA_FACTOR * sigma_x * sigma_y, 1.0) + psf_aperture_radius = max(2.5 * sigma, 1.0) + psf_annulus_width = max(5.0 * sigma, 3.0) + try: + sky_geometry = resolve_sky_annulus_geometry( + psf_aperture_radius, + psf_annulus_width, + psf_sigma=sigma, + ) + _, sigmabg, n_sky = skybg_phot( + data, + star_index, + xc, + yc, + sky_geometry['inner_radius'], + sky_geometry['annulus_width'], + fast_mode=fast_mode, + ) + except Exception: + sigmabg = np.nan + n_sky = np.nan + + return compute_photometry_noise_budget( + flux, + sigmabg, + effective_pixels, + n_sky, + exposure_s=exposure_s, + airmass=airmass, + noise_config=noise_config, + ) + + +def store_psf_noise_budget(psf_noise_data, key, frame_index, budget): + if not isinstance(psf_noise_data, dict) or key not in psf_noise_data: + return + psf_noise_data[key][frame_index] = budget.get('total', np.nan) + for component in NOISE_BUDGET_COMPONENT_KEYS: + component_key = f"{key}_noise_{component}" + if component_key in psf_noise_data: + psf_noise_data[component_key][frame_index] = budget.get(component, np.nan) + + +def psf_flux_data_source(psf_data, psf_flux_data=None): + if isinstance(psf_flux_data, dict) and 'target' in psf_flux_data: + return psf_flux_data + return psf_data + + +def is_fast_aperture_mask_enabled(config_value): + if config_value is None: + return False + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'fast', 'center', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'exact', 'off'): + return False + + log_info("Warning: Invalid 'Fast Aperture Mask (y/n)' value; using exact mode.", warn=True) + return False + + +def is_comp_star_required(config_value): + if config_value is None: + return True + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off'): + return False + + log_info("Warning: Invalid 'require_comp_star' value; requiring a comparison star.", warn=True) + return True + + +def is_target_driven_comp_selection_enabled(config_value): + if config_value is None: + return False + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off'): + return False + + log_info("Warning: Invalid target-driven comparison selection value; using comp-driven selection.", warn=True) + return False + + +def should_skip_low_comparison_coverage_rejection(config_value): + if config_value is None: + return False + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info("Warning: Invalid 'skip_low_comparison_coverage_rejection' value; keeping coverage rejection enabled.", + warn=True) + return False + + +def should_fit_lightcurve_to_every_comparison_candidate(config_value): + if config_value is None: + return False + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info("Warning: Invalid 'fit_lightcurve_to_every_comparison_candidate' value; defaulting to disabled.", + warn=True) + return False + + +def should_use_automatic_optimal_calibration_selector(config_value): + return parse_bool_config_value( + config_value, + False, + 'automatic_optimal_calibration_selector', + ) + + +def should_use_ensemble_photometry_rather_than_single_comp(config_value): + return parse_bool_config_value( + config_value, + False, + 'use_ensemble_photometry_rather_than_single_comp', + ) + + +def should_use_ensemble_photometry_for_stellar_variability(config_value): + return parse_bool_config_value( + config_value, + STELLAR_VARIABILITY_ENSEMBLE_DEFAULT, + 'use_ensemble_photometry_for_stellar_variability', + ) + + +def parse_ensemble_comparison_limit(config_value, config_key, default_value): + if config_value is None or config_value == '': + return default_value + try: + count = int(float(config_value)) + except (TypeError, ValueError): + log_info( + f"Warning: Invalid '{config_key}' value; defaulting to {default_value}.", + warn=True, + ) + return default_value + if count < STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS: + log_info( + f"'{config_key}' must be at least {STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS}; " + f"defaulting to {default_value}.", + warn=True, + ) + return default_value + return count + + +def parse_maximum_number_of_ensemble_comparisons_for_transit(config_value): + return parse_ensemble_comparison_limit( + config_value, + 'maximum_number_of_ensemble_comparisons_for_transit', + TRANSIT_ENSEMBLE_MAX_COMPARISONS_DEFAULT, + ) + + +def parse_maximum_number_of_ensemble_comparisons_for_stellar_variability(config_value): + return parse_ensemble_comparison_limit( + config_value, + 'maximum_number_of_ensemble_comparisons_for_stellar_variability', + STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS, + ) + + +def should_photometer_fortuitous_variables(config_value): + return parse_bool_config_value( + config_value, + PHOTOMETER_FORTUITOUS_VARIABLES_DEFAULT, + 'photometer_fortuitous_variables', + ) + + +def should_use_single_comparison_for_fortuitous_variables(config_value): + return parse_bool_config_value( + config_value, + USE_SINGLE_COMPARISON_FOR_FORTUITOUS_VARIABLES_DEFAULT, + 'use_single_comparison_for_fortuitous_variables', + ) + + +def should_use_nextastro_vsx_cache_first(config_value): + return parse_bool_config_value( + config_value, + USE_NEXTASTRO_VSX_CACHE_FIRST_DEFAULT, + 'use_nextastro_vsx_cache_first', + ) + + +def parse_automatic_calibration_selector_count(config_value): + if config_value is None or config_value == '': + return AUTOMATIC_CALIBRATION_SELECTOR_DEFAULT_COUNT + try: + count = int(float(config_value)) + except (TypeError, ValueError): + log_info( + "Warning: Invalid 'automatic_optimal_calibration_selector_count' value; " + f"defaulting to {AUTOMATIC_CALIBRATION_SELECTOR_DEFAULT_COUNT}.", + warn=True, + ) + return AUTOMATIC_CALIBRATION_SELECTOR_DEFAULT_COUNT + if count < 1: + log_info( + "Warning: 'automatic_optimal_calibration_selector_count' must be at least 1; " + f"defaulting to {AUTOMATIC_CALIBRATION_SELECTOR_DEFAULT_COUNT}.", + warn=True, + ) + return AUTOMATIC_CALIBRATION_SELECTOR_DEFAULT_COUNT + return count + + +def should_use_sparse_posterior_live_point_retry(config_value): + if config_value is None: + return SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_DEFAULT + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'use_sparse_posterior_live_point_retry' value; " + "defaulting to enabled.", + warn=True, + ) + return SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_DEFAULT + + +def should_run_fast_ultranest_before_final_run(config_value): + if config_value is None: + return FAST_ULTRANEST_BEFORE_FINAL_RUN_DEFAULT + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'run fast ultranest before final run' value; " + "defaulting to enabled.", + warn=True, + ) + return FAST_ULTRANEST_BEFORE_FINAL_RUN_DEFAULT + + +def should_run_final_residual_rejection(config_value): + if config_value is None: + return FINAL_RESIDUAL_REJECTION_DEFAULT + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'run final residual rejection and extra ultranest run' value; " + "defaulting to enabled.", + warn=True, + ) + return FINAL_RESIDUAL_REJECTION_DEFAULT + + +def should_use_legacy_psf_flux_mode(config_value): + if config_value is None: + return LEGACY_PSF_FLUX_MODE_DEFAULT + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on', 'legacy'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'use_legacy_psf_flux' value; " + "defaulting to modern PSF flux mode.", + warn=True, + ) + return LEGACY_PSF_FLUX_MODE_DEFAULT + + +def should_run_final_fit_phase_residual_clip(config_value): + if config_value is None: + return FINAL_FIT_PHASE_RESIDUAL_CLIP_DEFAULT + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'run_final_fit_phase_residual_clip' value; " + "defaulting to enabled.", + warn=True, + ) + return FINAL_FIT_PHASE_RESIDUAL_CLIP_DEFAULT + + +def psf_seed_track_directory_from_config(config_value): + if config_value is None: + return None + if isinstance(config_value, bool): + return None + if isinstance(config_value, (int, float)): + return None + if isinstance(config_value, str): + value = config_value.strip() + if value.lower() in ('', 'n', 'no', 'false', '0', 'off', 'none', 'null'): + return None + return value + return str(config_value) + + +def configure_sparse_posterior_live_point_retry(config_value): + enabled = should_use_sparse_posterior_live_point_retry(config_value) + os.environ[SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_ENV] = "1" if enabled else "0" + return enabled + + +def should_pick_comparison_by_eebls_snr(config_value): + if config_value is None: + return True + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'pick_comparison_by_eebls_snr' value; " + "keeping EEBLS SNR comparison selection enabled.", + warn=True, + ) + return True + + +def should_use_deviation_from_expected_transit_in_qc(config_value): + if config_value is None: + return TRANSIT_QC_USE_DEVIATION_FROM_EXPECTED_DEFAULT + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'use_deviation_from_expected_transit_in_qc' value; " + "keeping expected-value deviation QC enabled.", + warn=True, + ) + return TRANSIT_QC_USE_DEVIATION_FROM_EXPECTED_DEFAULT + + +def parse_deviation_from_expected_transit_in_qc_sigma(config_value): + if config_value is None: + return TRANSIT_QC_DEVIATION_SIGMA_DEFAULT + + try: + sigma_value = float(config_value) + except (TypeError, ValueError): + log_info( + "Warning: Invalid 'deviation_from_expected_transit_in_qc_sigma' value; using the default 5 sigma.", + warn=True, + ) + return TRANSIT_QC_DEVIATION_SIGMA_DEFAULT + + if not np.isfinite(sigma_value) or sigma_value <= 0: + log_info( + "Warning: Non-positive 'deviation_from_expected_transit_in_qc_sigma' value; using the default 5 sigma.", + warn=True, + ) + return TRANSIT_QC_DEVIATION_SIGMA_DEFAULT + return float(sigma_value) + + +def should_exit_at_first_qc_pass_solution(config_value): + if config_value is None: + return True + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'exit_at_first_qc_pass_solution' value; " + "defaulting to exit at the first QC PASS solution.", + warn=True, + ) + return True + + +def parse_ultranest_min_num_live_points(config_value): + if config_value is None: + return ULTRANEST_MIN_NUM_LIVE_POINTS_DEFAULT + + if isinstance(config_value, str) and config_value.strip() == "": + return ULTRANEST_MIN_NUM_LIVE_POINTS_DEFAULT + + try: + live_points = int(float(str(config_value).strip())) + except (TypeError, ValueError): + log_info( + "Warning: Invalid 'minimum number of live points for ultranest' value; " + f"defaulting to {ULTRANEST_MIN_NUM_LIVE_POINTS_DEFAULT}.", + warn=True, + ) + return ULTRANEST_MIN_NUM_LIVE_POINTS_DEFAULT + + if live_points <= 0: + log_info( + "Warning: 'minimum number of live points for ultranest' must be positive; " + f"defaulting to {ULTRANEST_MIN_NUM_LIVE_POINTS_DEFAULT}.", + warn=True, + ) + return ULTRANEST_MIN_NUM_LIVE_POINTS_DEFAULT + + return live_points + + +def configure_ultranest_min_num_live_points(config_value): + live_points = parse_ultranest_min_num_live_points(config_value) + os.environ[ULTRANEST_MIN_NUM_LIVE_POINTS_ENV] = str(live_points) + return live_points + + +def parse_rprs_search_bound_max(config_value): + if config_value is None: + return RPRS_SEARCH_BOUND_MAX_DEFAULT + + if isinstance(config_value, str) and config_value.strip() == "": + return RPRS_SEARCH_BOUND_MAX_DEFAULT + + try: + max_bound = float(str(config_value).strip()) + except (TypeError, ValueError): + log_info( + "Warning: Invalid 'rprs_search_bound_max' value; " + f"defaulting to {RPRS_SEARCH_BOUND_MAX_DEFAULT:.3f}.", + warn=True, + ) + return RPRS_SEARCH_BOUND_MAX_DEFAULT + + if not np.isfinite(max_bound) or max_bound <= RPRS_SEARCH_BOUND_MIN: + log_info( + "Warning: 'rprs_search_bound_max' must be finite and positive; " + f"defaulting to {RPRS_SEARCH_BOUND_MAX_DEFAULT:.3f}.", + warn=True, + ) + return RPRS_SEARCH_BOUND_MAX_DEFAULT + + if max_bound > RPRS_SEARCH_BOUND_ABSOLUTE_MAX: + log_info( + "Warning: 'rprs_search_bound_max' exceeds the absolute safety ceiling " + f"of {RPRS_SEARCH_BOUND_ABSOLUTE_MAX:.3f}; clamping to that ceiling.", + warn=True, + ) + return RPRS_SEARCH_BOUND_ABSOLUTE_MAX + + return float(max_bound) + + +def configure_rprs_search_bound_max(config_value): + global RPRS_SEARCH_BOUND_MAX + RPRS_SEARCH_BOUND_MAX = parse_rprs_search_bound_max(config_value) + return RPRS_SEARCH_BOUND_MAX + + +def parse_bool_config_value(config_value, default, option_name): + if config_value is None: + return default + parsed = coerce_boolean_config_value(config_value) + if parsed is not None: + return parsed + + default_text = "enabled" if default else "disabled" + log_info( + f"Warning: Invalid '{option_name}' value; defaulting to {default_text}.", + warn=True, + ) + return default + + +def parse_range_restriction_percentage(config_value, default, option_name): + if config_value is None: + return default + if isinstance(config_value, str) and config_value.strip() == "": + return default + + try: + percentage = float(str(config_value).strip().rstrip('%')) + except (TypeError, ValueError): + log_info( + f"Warning: Invalid '{option_name}' value; defaulting to {default:.1f}%.", + warn=True, + ) + return default + + if not np.isfinite(percentage) or percentage <= 0: + log_info( + f"Warning: '{option_name}' must be finite and positive; defaulting to {default:.1f}%.", + warn=True, + ) + return default + + return float(percentage) + + +def should_restrict_rprs_range(config_value): + return parse_bool_config_value( + config_value, + RPRS_RANGE_RESTRICTION_DEFAULT, + 'restrict_Rp/Rs_range', + ) + + +def should_use_prior_rprs_when_posterior_pinned(config_value): + return parse_bool_config_value( + config_value, + RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR_DEFAULT, + 'use_prior_Rp/Rs_when_posterior_pinned', + ) + + +def should_restrict_ars_range(config_value): + return parse_bool_config_value( + config_value, + ARS_RANGE_RESTRICTION_DEFAULT, + 'restrict_a/Rs_range', + ) + + +def configure_rprs_range_restriction(enabled_value, percentage_value): + global RPRS_RANGE_RESTRICTION_ENABLED, RPRS_RANGE_RESTRICTION_PERCENTAGE + RPRS_RANGE_RESTRICTION_ENABLED = should_restrict_rprs_range(enabled_value) + RPRS_RANGE_RESTRICTION_PERCENTAGE = parse_range_restriction_percentage( + percentage_value, + RPRS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT, + 'restrict_Rp/Rs_range_percentage', + ) + return RPRS_RANGE_RESTRICTION_ENABLED, RPRS_RANGE_RESTRICTION_PERCENTAGE + + +def configure_prior_rprs_fallback_on_pinned_posterior(config_value): + global RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR + RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR = should_use_prior_rprs_when_posterior_pinned(config_value) + return RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR + + +def configure_ars_range_restriction(enabled_value, percentage_value): + global ARS_RANGE_RESTRICTION_ENABLED, ARS_RANGE_RESTRICTION_PERCENTAGE + ARS_RANGE_RESTRICTION_ENABLED = should_restrict_ars_range(enabled_value) + ARS_RANGE_RESTRICTION_PERCENTAGE = parse_range_restriction_percentage( + percentage_value, + ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT, + 'restrict_a/Rs_range_percentage', + ) + return ARS_RANGE_RESTRICTION_ENABLED, ARS_RANGE_RESTRICTION_PERCENTAGE + + +def log_ultranest_mpi_status(): + status = get_mpi_status() + size = int(status.get("size") or 1) + rank = int(status.get("rank") or 0) + if size <= 1 or rank != 0: + return status + + if status.get("available"): + log_info(f"UltraNest MPI mode detected: {size} process(es).") + else: + log_info( + "Warning: MPI launch detected, but mpi4py is unavailable; " + "UltraNest cannot coordinate MPI workers until mpi4py is installed.", + warn=True, + ) + return status + + +def validate_ultranest_mpi_runtime(): + status = get_mpi_status() + size = int(status.get("size") or 1) + if size <= 1: + return status + + message = ( + "EXOTIC was launched under MPI, which duplicates the full reduction on every rank. " + "Start EXOTIC once and set EXOTIC_ULTRANEST_WORKERS to control UltraNest CPU parallelism." + ) + if int(status.get("rank") or 0) == 0: + log_info(f"Error: {message}", error=True) + raise RuntimeError(message) + + +def configure_windows_multiprocessing_main_spec(): + if sys.platform != "win32": + return False + + configured = False + spawn_executable = _windows_python_spawn_executable() + if spawn_executable: + multiprocessing.set_executable(spawn_executable) + if getattr(sys, "frozen", False): + sys.frozen = False + configured = True + + main_module = sys.modules.get("__main__") + if main_module is None: + return configured + + main_file = getattr(main_module, "__file__", None) + if not main_file or os.path.basename(os.fspath(main_file)).lower() not in {"exotic.exe", "exotic-script.py"}: + return configured + + if getattr(main_module, "__spec__", None) is not None: + main_module.__spec__ = None + main_module.__file__ = None + if getattr(main_module, "__package__", None) is not None: + main_module.__package__ = None + + return True + + +def ProcessPoolExecutor(*args, **kwargs): + if sys.platform == "win32": + return ThreadPoolExecutor(*args, **kwargs) + return _ProcessPoolExecutor(*args, **kwargs) + + +def ImageProcessPoolExecutor(*args, **kwargs): + """Use real processes for CPU-bound image work, including on Windows. + + The general EXOTIC executor intentionally retains its Windows thread fallback + for GUI/fitter compatibility. Image alignment workers are module-level, + pickle-safe functions and benefit materially from bypassing the GIL, so use + a spawned process context for that narrower workload. + """ + if sys.platform == "win32": + kwargs.setdefault('mp_context', multiprocessing.get_context('spawn')) + return _ProcessPoolExecutor(*args, **kwargs) + + +def _windows_python_spawn_executable(): + candidates = [ + getattr(sys, "_base_executable", None), + sys.executable, + os.path.join(sys.exec_prefix, "python.exe"), + os.path.join(getattr(sys, "base_exec_prefix", sys.exec_prefix), "python.exe"), + ] + for candidate in candidates: + if not candidate: + continue + executable = os.fspath(candidate) + if os.path.basename(executable).lower() in {"python.exe", "pythonw.exe"}: + return executable + return None + + +def should_use_psf_photometry(config_value): + if config_value is None: + return True + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info("Warning: Invalid 'use_psf_photometry' value; keeping PSF photometry enabled.", warn=True) + return True + + +def should_use_aperture_photometry(config_value): + if config_value is None: + return True + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info("Warning: Invalid 'use_aperture_photometry' value; keeping aperture photometry enabled.", warn=True) + return True + + +def should_use_eebls_to_initialize_tmid_and_bounds(config_value): + if config_value is None: + return True + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'use_eebls_to_initialize_tmid_and_bounds' value; " + "keeping the EEBLS transit initializer enabled.", + warn=True, + ) + return True + + +def should_detect_bad_pixels_before_photometry(config_value): + if config_value is None: + return False + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'detect_bad_pixels_before_photometry' value; keeping bad-pixel precheck disabled.", + warn=True, + ) + return False + + +def get_multiprocess_bad_pixel_precheck_processes(config_value): + if config_value is None: + return None + if isinstance(config_value, bool): + if not config_value: + return None + return os.cpu_count() or 1 + if isinstance(config_value, (int, float)): + if np.isfinite(config_value) and int(config_value) > 0: + return int(config_value) + return None + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('', 'n', 'no', 'false', '0', 'off'): + return None + if normalized in ('y', 'yes', 'true', '1', 'on'): + return os.cpu_count() or 1 + try: + parsed = float(normalized) + except ValueError: + parsed = np.nan + if np.isfinite(parsed) and int(parsed) > 0: + return int(parsed) + + log_info( + "Warning: Invalid 'multiprocess_bad_pixel_precheck' value; keeping bad-pixel precheck multiprocessing disabled.", + warn=True, + ) + return None + + +def is_adaptive_aperture_mode_enabled(config_value): + if config_value is None: + return False + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info("Warning: Invalid 'use_adaptive_apertures' value; using fixed apertures.", warn=True) + return False + + +def should_use_aperture_corrections_and_full_image_fwhm(config_value): + if config_value is None: + return False + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'use_aperture_corrections_and_full_image_fwhm' value; " + "keeping aperture corrections and full-image FWHM estimation disabled.", + warn=True, + ) + return False + + +def should_reject_overexposed_stars(config_value): + if config_value is None: + return REJECT_OVEREXPOSED_STARS_DEFAULT + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'reject_overexposed_stars' value; keeping overexposed-star rejection enabled.", + warn=True, + ) + return REJECT_OVEREXPOSED_STARS_DEFAULT + + +def parse_saturation_value(config_value): + if config_value is None: + return SATURATION_VALUE_DEFAULT + if isinstance(config_value, str) and not config_value.strip(): + return SATURATION_VALUE_DEFAULT + try: + value = float(config_value) + except (TypeError, ValueError): + value = np.nan + if np.isfinite(value) and value > 0: + return float(value) + + log_info( + "Warning: Invalid 'saturation_value' value; using 65535 for overexposure rejection.", + warn=True, + ) + return SATURATION_VALUE_DEFAULT + + +def parse_saturation_value_adu(config_value): + return parse_saturation_value(config_value) + + +def header_value_case_insensitive(header, key): + if not header: + return None + try: + return header[key] + except Exception: + pass + key_lower = str(key).lower() + try: + items = header.items() + except Exception: + return None + for header_key, header_value in items: + if str(header_key).lower() == key_lower: + return header_value + return None + + +def microobservatory_saturation_value_from_header(header): + telescop = header_value_case_insensitive(header, 'TELESCOP') + telescop = header_scalar_value(telescop) + if telescop is None: + return None + return MICROOBSERVATORY_TELESCOP_SATURATION_VALUES.get(str(telescop).strip().lower()) + + +def saturation_value_from_header(header): + microobservatory_saturation = microobservatory_saturation_value_from_header(header) + if microobservatory_saturation is not None: + return microobservatory_saturation + + saturate = header_value_case_insensitive(header, 'SATURATE') + saturate = finite_header_float(saturate) + if saturate is not None and saturate > 0: + return float(saturate) + return None + + +def parse_overexposure_threshold_fraction(config_value): + if config_value is None: + return OVEREXPOSURE_THRESHOLD_FRACTION_DEFAULT + if isinstance(config_value, str) and not config_value.strip(): + return OVEREXPOSURE_THRESHOLD_FRACTION_DEFAULT + try: + value = float(config_value) + except (TypeError, ValueError): + value = np.nan + if np.isfinite(value) and 0 < value <= 1: + return float(value) + + log_info( + "Warning: Invalid 'overexposure_threshold_fraction' value; " + f"using {OVEREXPOSURE_THRESHOLD_FRACTION_DEFAULT:.1f} for overexposure rejection.", + warn=True, + ) + return OVEREXPOSURE_THRESHOLD_FRACTION_DEFAULT + + +def should_ignore_header_wcs(config_value): + if config_value is None: + return False + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info("Warning: Invalid 'Ignore WCS in Header and Do Manual Alignment? (y/n)' value; " + "using header WCS when available.", warn=True) + return False + + +def should_allow_pixel_alignment_fallback(config_value): + if config_value is None: + return True + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'allow_pixel_alignment_fallback' value; " + "allowing pixel-based image alignment when WCS coverage is incomplete.", + warn=True, + ) + return True + + +def should_prefer_pixel_values_over_wcs_for_target(config_value): + if config_value is None: + return False + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info("Warning: Invalid 'prefer_pixel_values_over_wcs_for_target' value; " + "using WCS target coordinates.", warn=True) + return False + + +def get_bad_wcs_threshold_fraction(config_value): + default_fraction = SPARSE_MISSING_WCS_DROP_THRESHOLD + default_percent = default_fraction * 100.0 + if config_value is None: + return default_fraction + + if isinstance(config_value, str): + normalized = config_value.strip() + if normalized == "": + return default_fraction + if normalized.endswith('%'): + normalized = normalized[:-1].strip() + else: + normalized = config_value + + try: + threshold_percent = float(normalized) + except (TypeError, ValueError): + log_info(f"Warning: Invalid 'bad_wcs_threshold_percent' value; using default {default_percent:g}%.", + warn=True) + return default_fraction + + if not np.isfinite(threshold_percent) or threshold_percent < 0 or threshold_percent > 100: + log_info(f"Warning: Invalid 'bad_wcs_threshold_percent' value; using default {default_percent:g}%.", + warn=True) + return default_fraction + + return threshold_percent / 100.0 + + +def get_pointing_rejection_sigma(config_value): + if config_value is None: + return None + + if isinstance(config_value, str): + normalized = config_value.strip() + if normalized.lower() in ("", "0", "n", "no", "false", "off"): + return None + else: + normalized = config_value + + try: + sigma = float(normalized) + except (TypeError, ValueError): + log_info( + "Warning: Invalid 'pointing_rejection_sigma' value; disabling pointing precheck.", + warn=True, + ) + return None + + if not np.isfinite(sigma): + log_info( + "Warning: Invalid 'pointing_rejection_sigma' value; disabling pointing precheck.", + warn=True, + ) + return None + + if sigma < 0: + log_info( + "Warning: Invalid 'pointing_rejection_sigma' value; disabling pointing precheck.", + warn=True, + ) + return None + + if sigma == 0: + return None + + return sigma + + +def is_vertical_flux_normalization_disabled(config_value): + if config_value is None: + return False + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info("Warning: Invalid 'disable vertical flux normalization' value; using default enabled normalization.", warn=True) + return False + + +def should_run_stellar_variability_only(config_value): + if config_value is None: + return STELLAR_VARIABILITY_ONLY_DEFAULT + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'stellar_variability_only' value; using default false.", + warn=True, + ) + return STELLAR_VARIABILITY_ONLY_DEFAULT + + +def is_out_of_transit_baseline_detrending_enabled(config_value): + if config_value is None: + return True + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'detrend_on_outoftransit_baseline' value; using default enabled setting.", + warn=True, + ) + return True + + +def get_final_fit_baseline_duration_multiplier(config_value): + if config_value is None: + return FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT + if isinstance(config_value, (int, float)) and np.isfinite(config_value) and config_value >= 0: + return float(config_value) + if isinstance(config_value, str): + normalized = config_value.strip() + if normalized == "": + return FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT + try: + parsed = float(normalized) + except ValueError: + parsed = np.nan + if np.isfinite(parsed) and parsed >= 0: + return float(parsed) + + log_info( + "Warning: Invalid 'final_fit_baseline_duration_multiplier' value; " + f"using default {FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT:.1f}.", + warn=True, + ) + return FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT + + +def estimate_transit_duration_from_prior_geometry(prior): + try: + period = float(prior['per']) + rprs = float(prior['rprs']) + ars = float(prior['ars']) + inc = float(prior['inc']) + except (KeyError, TypeError, ValueError): + return np.nan + + if ( + not np.isfinite(period) or period <= 0 + or not np.isfinite(rprs) or rprs < 0 + or not np.isfinite(ars) or ars <= 0 + or not np.isfinite(inc) + ): + return np.nan + + ecc = prior.get('ecc', 0.0) + omega = np.deg2rad(prior.get('omega', 0.0)) + sin_inc = np.sin(np.deg2rad(inc)) + if not np.isfinite(sin_inc) or sin_inc <= 0: + return np.nan + + impact_scale = ars * (1.0 - ecc ** 2) / max(np.finfo(float).eps, 1.0 + ecc * np.sin(omega)) + impact_parameter = impact_scale * np.cos(np.deg2rad(inc)) + chord_sq = (1.0 + rprs) ** 2 - impact_parameter ** 2 + if not np.isfinite(chord_sq) or chord_sq <= 0 or not np.isfinite(impact_scale) or impact_scale <= 0: + return np.nan + + argument = np.sqrt(chord_sq) / (ars * sin_inc) + argument = float(np.clip(argument, -1.0, 1.0)) + eccentric_speed_factor = np.sqrt(1.0 - ecc ** 2) / max(np.finfo(float).eps, 1.0 + ecc * np.sin(omega)) + duration = (period / np.pi) * np.arcsin(argument) * eccentric_speed_factor + return float(duration) if np.isfinite(duration) and duration > 0 else np.nan + + +def stellar_variability_transit_prior_from_planet_dict(p_dict): + return { + 'per': p_dict.get('pPer'), + 'rprs': p_dict.get('rprs'), + 'ars': p_dict.get('aRs'), + 'inc': p_dict.get('inc'), + 'ecc': p_dict.get('ecc', 0.0), + 'omega': p_dict.get('omega', 0.0), + } + + +def stellar_variability_out_of_transit_mask(times, p_dict): + times = np.asarray(times, dtype=float) + keep = np.isfinite(times) + summary = { + 'applied': False, + 'input_point_count': int(np.count_nonzero(np.isfinite(times))), + 'kept_point_count': int(np.count_nonzero(keep)), + 'rejected_point_count': 0, + 'duration_days': np.nan, + 'start_ingress_to_end_egress_days': np.nan, + 'period_days': np.nan, + 'reference_tmid': np.nan, + 'note': "Transit-window exclusion was not applied.", + } + if times.size == 0: + summary['note'] = "Transit-window exclusion skipped; no light-curve points were available." + return keep, summary + + try: + period = float(p_dict.get('pPer')) + reference_tmid = float(p_dict.get('midT')) + except (TypeError, ValueError): + period = np.nan + reference_tmid = np.nan + + duration = estimate_transit_duration_from_prior_geometry( + stellar_variability_transit_prior_from_planet_dict(p_dict) + ) + summary.update({ + 'duration_days': duration, + 'start_ingress_to_end_egress_days': duration, + 'period_days': period, + 'reference_tmid': reference_tmid, + }) + if ( + not np.isfinite(period) + or period <= 0 + or not np.isfinite(reference_tmid) + or not np.isfinite(duration) + or duration <= 0 + ): + summary['note'] = ( + "Transit-window exclusion skipped; EXOTIC could not estimate a finite ingress-to-egress " + "window from the supplied planetary parameters." + ) + return keep, summary + + epochs = np.rint((times - reference_tmid) / period) + nearest_tmid = reference_tmid + epochs * period + in_transit = keep & (np.abs(times - nearest_tmid) <= 0.5 * duration) + keep = keep & ~in_transit + summary.update({ + 'applied': bool(np.any(in_transit)), + 'kept_point_count': int(np.count_nonzero(keep)), + 'rejected_point_count': int(np.count_nonzero(in_transit)), + 'note': ( + f"Excluded {int(np.count_nonzero(in_transit))} point(s) inside the predicted " + "start-ingress to end-egress transit window before stellar-variability analysis." + ), + }) + return keep, summary + + +def _cacheable_duration_prior_scalar(value): + try: + numeric_value = float(value) + except (TypeError, ValueError): + return None + return None if not np.isfinite(numeric_value) else float(numeric_value) + + +def _duration_prior_scalar_from_cache(value): + return np.nan if value is None else float(value) + + +@lru_cache(maxsize=128) +def _cached_single_transit_duration_prior( + period, + rprs, + ars, + inc, + ecc, + omega, + period_unc, + rprs_unc, + ars_unc, + inc_unc, +): + prior = { + 'per': _duration_prior_scalar_from_cache(period), + 'rprs': _duration_prior_scalar_from_cache(rprs), + 'ars': _duration_prior_scalar_from_cache(ars), + 'inc': _duration_prior_scalar_from_cache(inc), + 'ecc': _duration_prior_scalar_from_cache(ecc), + 'omega': _duration_prior_scalar_from_cache(omega), + } + expected_duration = estimate_transit_duration_from_prior_geometry(prior) + if not np.isfinite(expected_duration) or expected_duration <= 0: + return { + 'applied': False, + 'expected_duration': np.nan, + 'sigma_log_duration': np.nan, + 'relative_sigma': np.nan, + 'source': 'unavailable', + 'sample_count': 0, + 'note': ( + "Not applied; could not estimate a physical transit duration from the published single-transit priors." + ), + } + + period_unc = _duration_prior_scalar_from_cache(period_unc) + rprs_unc = _duration_prior_scalar_from_cache(rprs_unc) + ars_unc = _duration_prior_scalar_from_cache(ars_unc) + inc_unc = _duration_prior_scalar_from_cache(inc_unc) + + fallback_sigma_log = float(np.log1p(DURATION_PRIOR_FALLBACK_RELATIVE_SIGMA)) + min_sigma_log = float(np.log1p(DURATION_PRIOR_MIN_RELATIVE_SIGMA)) + sigma_log_duration = fallback_sigma_log + sample_count = 0 + source = "fallback relative width" + + if any( + value is not None and value > 0 + for value in (period_unc, rprs_unc, ars_unc, inc_unc) + ): + rng = np.random.default_rng(0) + sample_draw_count = int(max(DURATION_PRIOR_MONTE_CARLO_SAMPLES, 1)) + period_draws = np.full(sample_draw_count, prior['per'], dtype=float) + rprs_draws = np.full(sample_draw_count, prior['rprs'], dtype=float) + ars_draws = np.full(sample_draw_count, prior['ars'], dtype=float) + inc_draws = np.full(sample_draw_count, prior['inc'], dtype=float) + + if period_unc is not None and period_unc > 0: + period_draws = rng.normal(prior['per'], period_unc, sample_draw_count) + if rprs_unc is not None and rprs_unc > 0: + rprs_draws = rng.normal(prior['rprs'], rprs_unc, sample_draw_count) + if ars_unc is not None and ars_unc > 0: + ars_draws = rng.normal(prior['ars'], ars_unc, sample_draw_count) + if inc_unc is not None and inc_unc > 0: + inc_draws = rng.normal(prior['inc'], inc_unc, sample_draw_count) + + inc_draws = np.clip(inc_draws, 1e-6, 89.999999) + durations = np.full(sample_draw_count, np.nan, dtype=float) + for index in range(sample_draw_count): + durations[index] = estimate_transit_duration_from_prior_geometry({ + 'per': period_draws[index], + 'rprs': rprs_draws[index], + 'ars': ars_draws[index], + 'inc': inc_draws[index], + 'ecc': prior['ecc'], + 'omega': prior['omega'], + }) + + valid_durations = durations[np.isfinite(durations) & (durations > 0)] + sample_count = int(valid_durations.size) + if valid_durations.size >= DURATION_PRIOR_MIN_VALID_MONTE_CARLO_SAMPLES: + log_offsets = np.log(valid_durations / expected_duration) + lower_offset, upper_offset = np.nanpercentile(log_offsets, [16, 84]) + sigma_log_duration = float(max(0.5 * (upper_offset - lower_offset), min_sigma_log)) + source = "published geometry uncertainties" + else: + source = "fallback relative width (insufficient valid uncertainty samples)" + else: + source = "fallback relative width (missing published geometry uncertainties)" + + sigma_log_duration = float(max(sigma_log_duration, min_sigma_log)) + relative_sigma = float(np.expm1(sigma_log_duration)) + note = ( + f"Applied; expected duration={expected_duration:.6f} day(s) with an approximate 1-sigma width of " + f"{relative_sigma * 100.0:.1f}% from {source}" + ) + if sample_count > 0: + note += f" ({sample_count} propagated sample(s))." + else: + note += "." + + return { + 'applied': True, + 'expected_duration': float(expected_duration), + 'sigma_log_duration': float(sigma_log_duration), + 'relative_sigma': relative_sigma, + 'source': source, + 'sample_count': sample_count, + 'note': note, + } + + +def build_single_transit_duration_prior(planet_dict): + if not isinstance(planet_dict, dict): + return { + 'applied': False, + 'expected_duration': np.nan, + 'sigma_log_duration': np.nan, + 'relative_sigma': np.nan, + 'source': 'unavailable', + 'sample_count': 0, + 'note': "Not applied; missing published single-transit planet metadata.", + } + + return dict( + _cached_single_transit_duration_prior( + _cacheable_duration_prior_scalar(planet_dict.get('pPer', np.nan)), + _cacheable_duration_prior_scalar(planet_dict.get('rprs', np.nan)), + _cacheable_duration_prior_scalar(planet_dict.get('aRs', np.nan)), + _cacheable_duration_prior_scalar(planet_dict.get('inc', np.nan)), + _cacheable_duration_prior_scalar(planet_dict.get('ecc', 0.0)), + _cacheable_duration_prior_scalar(planet_dict.get('omega', 0.0)), + _cacheable_duration_prior_scalar(planet_dict.get('pPerUnc', np.nan)), + _cacheable_duration_prior_scalar(planet_dict.get('rprsUnc', np.nan)), + _cacheable_duration_prior_scalar(planet_dict.get('aRsUnc', np.nan)), + _cacheable_duration_prior_scalar(planet_dict.get('incUnc', np.nan)), + ) + ) + + +def estimate_ephemeris_tmid_and_bounds( + times, + prior_tmid, + period, + midt_unc, + per_unc, + expected_duration=np.nan, + sigma_multiplier=25.0, +): + summary = { + 'method': 'ephemeris', + 'applied': False, + 'tmid': float(prior_tmid) if np.isfinite(prior_tmid) else np.nan, + 'uncertainty': np.nan, + 'bounds': [np.nan, np.nan], + 'cycle_index': np.nan, + 'propagated_half_width': np.nan, + 'half_width': np.nan, + 'observations_bracket_expected_transit': False, + 'duration_capped': False, + 'observed_window_capped': False, + 'note': 'Using ephemeris-derived Tmid bounds.', + } + + try: + prior_tmid = float(prior_tmid) + period = float(period) + midt_unc = float(midt_unc) + per_unc = float(per_unc) + sigma_multiplier = float(sigma_multiplier) + except (TypeError, ValueError): + summary['note'] = 'Using ephemeris-derived Tmid bounds with invalid prior metadata.' + return summary + + if not np.isfinite(prior_tmid) or not np.isfinite(period) or period <= 0: + summary['note'] = 'Using ephemeris-derived Tmid bounds with invalid Tmid/period metadata.' + return summary + + valid_times = np.asarray(times, dtype=float) + valid_times = valid_times[np.isfinite(valid_times)] + if valid_times.size == 0: + summary['bounds'] = [prior_tmid, prior_tmid] + summary['note'] = 'Using ephemeris-derived Tmid bounds with no finite observation times.' + return summary + + phases = (valid_times - prior_tmid) / period + # Select the transit epoch nearest the bulk of the data. floor(phases).max() + # picked the transit at-or-before the last valid frame, which reports Tmid one + # full period early whenever the true mid falls after the last surviving frame + # (ingress-only partial transits, the common fixed-window case); the periodic + # transit model then fits perfectly at the wrong epoch. See issue #1387. + cycle_index = float(np.round(np.median(phases))) + tmid = float(prior_tmid + cycle_index * period) + propagated_uncertainty = np.sqrt(midt_unc ** 2 + (cycle_index * per_unc) ** 2) + + propagated_half_width = np.abs(sigma_multiplier * midt_unc + cycle_index * sigma_multiplier * per_unc) + max_half_width = 0.25 * period + if not np.isfinite(propagated_half_width) or propagated_half_width <= 0: + half_width = max_half_width + propagated_half_width = np.nan + else: + half_width = min(float(propagated_half_width), max_half_width) + + cadence = np.nan + if valid_times.size > 1: + cadence = np.nanmedian(np.diff(np.sort(valid_times))) + + lower = float(tmid - half_width) + upper = float(tmid + half_width) + if np.isfinite(expected_duration) and expected_duration > 0: + coverage_margin = 0.5 * float(expected_duration) + if np.isfinite(cadence) and cadence > 0: + coverage_margin = max(coverage_margin, 3.0 * cadence) + + pre_points = int(np.count_nonzero(valid_times < tmid - coverage_margin)) + post_points = int(np.count_nonzero(valid_times > tmid + coverage_margin)) + bracketed = pre_points > 0 and post_points > 0 + summary['observations_bracket_expected_transit'] = bracketed + + cadence_floor = 0.0 + if np.isfinite(cadence) and cadence > 0: + cadence_floor = 5.0 * cadence + duration_cap = max( + EPHEMERIS_BRACKETED_TMID_HALF_WIDTH_DURATION_MULTIPLIER * float(expected_duration), + cadence_floor, + ) + if bracketed and np.isfinite(duration_cap) and duration_cap > 0 and duration_cap < half_width: + half_width = float(duration_cap) + summary['duration_capped'] = True + lower = float(tmid - half_width) + upper = float(tmid + half_width) + + if bracketed: + observed_lower = float(np.nanmin(valid_times) + 0.5 * float(expected_duration)) + observed_upper = float(np.nanmax(valid_times) - 0.5 * float(expected_duration)) + if ( + np.isfinite(observed_lower) + and np.isfinite(observed_upper) + and observed_upper > observed_lower + ): + tightened_lower = max(lower, observed_lower) + tightened_upper = min(upper, observed_upper) + if tightened_upper > tightened_lower and ( + tightened_lower > lower + 1e-12 or tightened_upper < upper - 1e-12 + ): + lower = float(tightened_lower) + upper = float(tightened_upper) + summary['observed_window_capped'] = True + + half_width = max(float(tmid - lower), float(upper - tmid)) + summary.update({ + 'tmid': tmid, + 'uncertainty': propagated_uncertainty, + 'bounds': [lower, upper], + 'cycle_index': cycle_index, + 'propagated_half_width': propagated_half_width, + 'half_width': half_width, + 'applied': summary['duration_capped'] or summary['observed_window_capped'], + }) + if summary['observed_window_capped']: + summary['note'] = ( + "Ephemeris-derived Tmid bounds were intersected with the observed time span needed to contain the " + f"full expected transit; using bounds=[{lower:.6f}, {upper:.6f}] instead of the wider propagated " + f"half-width {float(propagated_half_width):.6f} day(s)." + ) + elif summary['duration_capped']: + summary['note'] = ( + "Ephemeris-derived Tmid bounds were narrowed to the expected-transit timescale because the " + f"observations bracket the expected transit; using bounds=[{lower:.6f}, {upper:.6f}] " + f"instead of the wider propagated half-width {float(propagated_half_width):.6f} day(s)." + ) + else: + summary['note'] = f"Using ephemeris-derived Tmid bounds [{lower:.6f}, {upper:.6f}]." + + return summary + + +def estimate_tmid_and_bounds_with_eebls(times, flux_values, flux_errors, prior, fallback_bounds): + summary = { + 'method': 'ephemeris', + 'applied': False, + 'tmid': float(prior.get('tmid', np.nan)), + 'bounds': [float(fallback_bounds[0]), float(fallback_bounds[1])], + 'duration': np.nan, + 'depth': np.nan, + 'depth_snr': np.nan, + 'note': 'EEBLS transit initializer did not run.', + } + + times = np.asarray(times, dtype=float) + flux_values = np.asarray(flux_values, dtype=float) + flux_errors = np.asarray(flux_errors, dtype=float) + period = float(prior.get('per', np.nan)) + if not np.isfinite(period) or period <= 0: + summary['note'] = 'EEBLS transit initializer skipped: invalid orbital period.' + return summary + + valid = np.isfinite(times) & np.isfinite(flux_values) & (flux_values > 0) + if flux_errors.shape == flux_values.shape: + valid &= np.isfinite(flux_errors) & (flux_errors > 0) + else: + flux_errors = np.full_like(flux_values, np.nan, dtype=float) + + if np.count_nonzero(valid) < max(LIGHTCURVE_MIN_VALID_POINTS, EEBLS_MIN_VALID_POINTS): + summary['note'] = 'EEBLS transit initializer skipped: not enough valid points.' + return summary + + valid_times = np.asarray(times[valid], dtype=float) + valid_flux = np.asarray(flux_values[valid], dtype=float) + valid_errors = np.asarray(flux_errors[valid], dtype=float) + sort_index = np.argsort(valid_times) + fit_times = valid_times[sort_index] + fit_flux = valid_flux[sort_index] + fit_errors = valid_errors[sort_index] + cadence = np.nanmedian(np.diff(fit_times)) + if not np.isfinite(cadence) or cadence <= 0: + cadence = max(np.finfo(float).eps, 0.005 * period) + + x = fit_times - np.nanmedian(fit_times) + baseline = np.ones_like(fit_flux, dtype=float) + design = np.column_stack((np.ones_like(x), x)) + if np.count_nonzero(np.isfinite(x)) >= 2: + weights = np.ones_like(fit_flux, dtype=float) + finite_error_mask = np.isfinite(fit_errors) & (fit_errors > 0) + if np.any(finite_error_mask): + weights[finite_error_mask] = 1.0 / (fit_errors[finite_error_mask] ** 2) + weights[~finite_error_mask] = 0.0 + if not np.any(weights > 0): + weights = np.ones_like(fit_flux, dtype=float) + sqrt_weights = np.sqrt(weights) + try: + coeffs, _, _, _ = np.linalg.lstsq(design * sqrt_weights[:, None], fit_flux * sqrt_weights, rcond=None) + baseline = coeffs[0] + coeffs[1] * x + if not np.all(np.isfinite(baseline)) or np.any(baseline <= 0): + baseline = np.ones_like(fit_flux, dtype=float) + except np.linalg.LinAlgError: + baseline = np.ones_like(fit_flux, dtype=float) + + detrended_flux = fit_flux / baseline + detrended_flux /= np.nanmedian(detrended_flux) + detrended_errors = fit_errors / baseline + if not np.all(np.isfinite(detrended_errors)) or np.any(detrended_errors <= 0): + detrended_errors = None + + expected_duration = estimate_transit_duration_from_prior_geometry(prior) + if not np.isfinite(expected_duration) or expected_duration <= 0: + expected_duration = 0.05 * period + + min_duration = max(3.0 * cadence, EEBLS_DURATION_MIN_FRACTION * expected_duration) + max_duration = min(0.25 * period, max(min_duration * 1.5, EEBLS_DURATION_MAX_FRACTION * expected_duration)) + if not np.isfinite(min_duration) or not np.isfinite(max_duration) or max_duration <= 0 or min_duration > max_duration: + summary['note'] = 'EEBLS transit initializer skipped: invalid duration search grid.' + return summary + + durations = np.linspace(min_duration, max_duration, EEBLS_DURATION_GRID_SIZE) + durations = np.unique(durations[np.isfinite(durations) & (durations > 0)]) + if durations.size == 0: + summary['note'] = 'EEBLS transit initializer skipped: empty duration search grid.' + return summary + + try: + bls = BoxLeastSquares(fit_times, detrended_flux, dy=detrended_errors) + results = bls.power(period, durations, objective='snr') + except Exception as exc: + summary['note'] = f'EEBLS transit initializer failed: {type(exc).__name__}: {exc}' + return summary + + power = np.asarray(results.power, dtype=float) + if power.size == 0 or not np.any(np.isfinite(power)): + summary['note'] = 'EEBLS transit initializer skipped: no finite search power values were returned.' + return summary + + best_index = int(np.nanargmax(power)) + tmid = float(np.asarray(results.transit_time, dtype=float)[best_index]) + duration = float(np.asarray(results.duration, dtype=float)[best_index]) + depth = float(np.asarray(results.depth, dtype=float)[best_index]) + depth_snr = float(np.asarray(results.depth_snr, dtype=float)[best_index]) + if ( + not np.isfinite(tmid) + or not np.isfinite(duration) or duration <= 0 + or not np.isfinite(depth) or depth <= 0 + or not np.isfinite(depth_snr) or depth_snr <= 0 + ): + summary['note'] = 'EEBLS transit initializer skipped: the best-fitting transit candidate was not physical.' + return summary + + coverage_margin = max(0.5 * duration, 3.0 * cadence) + pre_points = int(np.count_nonzero(np.isfinite(fit_times) & (fit_times < tmid - coverage_margin))) + post_points = int(np.count_nonzero(np.isfinite(fit_times) & (fit_times > tmid + coverage_margin))) + if pre_points == 0 or post_points == 0: + summary.update({ + 'method': 'eebls', + 'applied': False, + 'tmid': tmid, + 'duration': duration, + 'depth': depth, + 'depth_snr': depth_snr, + }) + summary['note'] = ( + "EEBLS transit initializer found a box-like event, but it is not bracketed by data on both sides " + f"({pre_points} pre-point(s), {post_points} post-point(s)); keeping the EEBLS depth SNR only and " + "falling back to the non-EEBLS Tmid bounds." + ) + return summary + + duration_for_bounds = duration + if np.isfinite(expected_duration) and expected_duration > 0: + duration_for_bounds = max(duration_for_bounds, 0.75 * expected_duration) + half_width = min( + 0.25 * period, + max(EEBLS_TMID_HALF_WIDTH_DURATION_MULTIPLIER * duration_for_bounds, 5.0 * cadence), + ) + if not np.isfinite(half_width) or half_width <= 0: + summary['note'] = 'EEBLS transit initializer skipped: invalid Tmid search half-width.' + return summary + + summary.update({ + 'method': 'eebls', + 'applied': True, + 'tmid': tmid, + 'bounds': [float(tmid - half_width), float(tmid + half_width)], + 'duration': duration, + 'depth': depth, + 'depth_snr': depth_snr, + 'note': ( + "EEBLS transit initializer found a box-like transit candidate at " + f"Tmid={tmid:.6f} day(s) with duration={duration:.6f} day(s), depth={depth:.5f}, " + f"depth_snr={depth_snr:.2f}, and bounds=[{tmid - half_width:.6f}, {tmid + half_width:.6f}]." + ), + }) + return summary + + +def annotate_lightcurve_tmid_search(fit, summary): + if fit is None: + return + + fit.initial_tmid_search_method = summary.get('method') + fit.initial_tmid_search_applied = bool(summary.get('applied')) + fit.initial_tmid_search_tmid = summary.get('tmid') + fit.initial_tmid_search_uncertainty = summary.get('uncertainty') + fit.initial_tmid_search_bounds = summary.get('bounds') + fit.initial_tmid_search_duration = summary.get('duration') + fit.initial_tmid_search_depth = summary.get('depth') + fit.initial_tmid_search_depth_snr = summary.get('depth_snr') + fit.initial_tmid_search_note = summary.get('note') + + +def annotate_lightcurve_eebls_diagnostic(fit, summary): + if fit is None: + return + + summary = {} if summary is None else dict(summary) + fit.eebls_diagnostic_computed = bool(summary) + fit.eebls_diagnostic_method = summary.get('method') + fit.eebls_diagnostic_applied = bool(summary.get('applied')) + fit.eebls_diagnostic_tmid = summary.get('tmid') + fit.eebls_diagnostic_bounds = summary.get('bounds') + fit.eebls_diagnostic_duration = summary.get('duration') + fit.eebls_diagnostic_depth = summary.get('depth') + fit.eebls_diagnostic_depth_snr = summary.get('depth_snr') + fit.eebls_diagnostic_note = summary.get('note') + + +def extract_lightcurve_fit_eebls_snr(fit): + if fit is None: + return np.nan + + for attr_name in ('eebls_diagnostic_depth_snr', 'initial_tmid_search_depth_snr'): + value = getattr(fit, attr_name, np.nan) + try: + numeric_value = float(value) + except (TypeError, ValueError): + continue + if np.isfinite(numeric_value): + return numeric_value + + return np.nan + + +def ensure_lightcurve_fit_eebls_diagnostic(fit): + if fit is None: + return None + + existing_snr = extract_lightcurve_fit_eebls_snr(fit) + if np.isfinite(existing_snr): + return { + 'applied': True, + 'depth_snr': existing_snr, + 'note': 'Existing EEBLS diagnostic reused.', + } + + times = np.asarray(getattr(fit, 'time', np.array([])), dtype=float) + flux_values = np.asarray(getattr(fit, 'data', np.array([])), dtype=float) + if times.ndim != 1 or times.size < LIGHTCURVE_MIN_VALID_POINTS or flux_values.shape != times.shape: + return None + + dataerr_obj = getattr(fit, 'dataerr', None) + flux_errors = None if dataerr_obj is None else np.asarray(dataerr_obj, dtype=float) + if flux_errors is None or flux_errors.shape != times.shape: + flux_errors = np.full(times.shape, 1.0, dtype=float) + else: + finite_positive = np.isfinite(flux_errors) & (flux_errors > 0) + if not np.any(finite_positive): + flux_errors = np.full(times.shape, 1.0, dtype=float) + elif not np.all(finite_positive): + replacement = float(np.nanmedian(flux_errors[finite_positive])) + flux_errors = np.where(finite_positive, flux_errors, replacement) + + fit_prior = getattr(fit, 'prior', {}) or {} + parameters = getattr(fit, 'parameters', {}) or {} + period = fit_prior.get('per', parameters.get('per', np.nan)) + tmid = fit_prior.get('tmid', parameters.get('tmid', np.nan)) + try: + period = float(period) + tmid = float(tmid) + except (TypeError, ValueError): + return None + if not np.isfinite(period) or period <= 0 or not np.isfinite(tmid): + return None + + fallback_bounds = getattr(fit, 'initial_tmid_search_bounds', None) + if fallback_bounds is None: + fallback_bounds = getattr(fit, 'eebls_diagnostic_bounds', None) + if fallback_bounds is None: + bounds = getattr(fit, 'bounds', None) + fallback_bounds = bounds.get('tmid') if isinstance(bounds, dict) else None + + try: + lower, upper = [float(value) for value in np.asarray(fallback_bounds, dtype=float).reshape(-1)[:2]] + except (TypeError, ValueError, IndexError): + lower = float(tmid - 0.25 * period) + upper = float(tmid + 0.25 * period) + if not np.isfinite(lower) or not np.isfinite(upper) or upper <= lower: + lower = float(tmid - 0.25 * period) + upper = float(tmid + 0.25 * period) + + eebls_summary = estimate_tmid_and_bounds_with_eebls( + times, + flux_values, + flux_errors, + {'per': period, 'tmid': tmid}, + [lower, upper], + ) + annotate_lightcurve_eebls_diagnostic(fit, eebls_summary) + return eebls_summary + + +def should_use_impactparameter_rather_than_inclination_to_fit(config_value): + if config_value is None: + return True + if isinstance(config_value, bool): + return config_value + if isinstance(config_value, (int, float)): + return bool(config_value) + if isinstance(config_value, str): + normalized = config_value.strip().lower() + if normalized in ('y', 'yes', 'true', '1', 'on'): + return True + if normalized in ('n', 'no', 'false', '0', 'off', ''): + return False + + log_info( + "Warning: Invalid 'use_impactparameter_rather_than_inclination_to_fit' value; " + "using impact parameter for nested fitting.", + warn=True, + ) + return True + + +def apply_vertical_flux_normalization_bound(prior, bounds, flux_values, disabled): + finite_flux = np.asarray(flux_values, dtype=float) + finite_flux = finite_flux[np.isfinite(finite_flux) & (finite_flux > 0)] + baseline_guess = 1.0 if finite_flux.size == 0 else float(np.nanmedian(finite_flux)) + if not np.isfinite(baseline_guess) or baseline_guess <= 0: + baseline_guess = 1.0 + + prior['a0'] = baseline_guess + prior['a1'] = baseline_guess + + if not disabled: + # Some paths deliver an approximately unity-normalized light curve, while + # others still carry an arbitrary positive baseline. Keep the legacy + # near-unity bound only when the working series is already close to 1. + if 0.95 <= baseline_guess <= 1.05: + bounds['a0'] = [0.95, 1.05] + else: + lower = max(np.finfo(float).eps, baseline_guess * 0.75) + upper = baseline_guess * 1.25 + bounds['a0'] = [lower, upper] + + +def ensure_pre_final_ultranest_baseline_bounds(prior, bounds, flux_values, fit_a2=True): + finite_flux = np.asarray(flux_values, dtype=float) + finite_flux = finite_flux[np.isfinite(finite_flux) & (finite_flux > 0)] + baseline_guess = prior.get('a0', prior.get('a1', np.nan)) + try: + baseline_guess = float(baseline_guess) + except (TypeError, ValueError): + baseline_guess = np.nan + if not np.isfinite(baseline_guess) or baseline_guess <= 0: + baseline_guess = 1.0 if finite_flux.size == 0 else float(np.nanmedian(finite_flux)) + if not np.isfinite(baseline_guess) or baseline_guess <= 0: + baseline_guess = 1.0 + + prior['a0'] = baseline_guess + prior['a1'] = baseline_guess + if 'a0' not in bounds and 'a1' not in bounds: + if 0.95 <= baseline_guess <= 1.05: + bounds['a0'] = [0.95, 1.05] + else: + lower = max(np.finfo(float).eps, baseline_guess * 0.75) + upper = baseline_guess * 1.25 + bounds['a0'] = [lower, upper] + + if fit_a2: + prior['a2'] = prior.get('a2', 0.0) + if 'a2' not in bounds: + bounds['a2'] = list(TRANSIT_QC_DEFAULT_A2_BOUNDS) + + +def _weighted_mean_with_fallback(values, weights=None): + values = np.asarray(values, dtype=float) + finite = np.isfinite(values) + if not np.any(finite): + return np.nan + + if weights is not None: + weights = np.asarray(weights, dtype=float) + valid_weights = finite & np.isfinite(weights) & (weights > 0) + if np.any(valid_weights): + return float(np.sum(values[valid_weights] * weights[valid_weights]) / np.sum(weights[valid_weights])) + + return float(np.nanmean(values[finite])) + + +def build_fast_ultranest_lightcurve_series( + times, + flux_values, + flux_errors, + airmass, + jd_times=None, + exposure_times_seconds=None, + max_points=FAST_ULTRANEST_MAX_BINNED_POINTS, + min_points_to_bin=FAST_ULTRANEST_MIN_POINTS_TO_BIN, +): + times = np.asarray(times, dtype=float) + flux_values = np.asarray(flux_values, dtype=float) + flux_errors = np.asarray(flux_errors, dtype=float) + airmass = np.asarray(airmass, dtype=float) + jd_array = None if jd_times is None else np.asarray(jd_times, dtype=float) + exposure_array = None if exposure_times_seconds is None else np.asarray(exposure_times_seconds, dtype=float) + + base_result = { + 'applied': False, + 'note': None, + 'time': times, + 'flux': flux_values, + 'unc': flux_errors, + 'airmass': airmass, + 'jd_times': jd_array, + 'exposure_times_seconds': exposure_array, + 'original_point_count': int(times.shape[0]), + 'binned_point_count': int(times.shape[0]), + 'bin_indices': None, + } + + if not (times.shape == flux_values.shape == flux_errors.shape == airmass.shape): + base_result['note'] = 'Skipped; light-curve arrays were not aligned for fast UltraNest binning.' + return base_result + if jd_array is not None and jd_array.shape != times.shape: + base_result['note'] = 'Skipped; JD timestamps were not aligned for fast UltraNest binning.' + return base_result + if exposure_array is not None and exposure_array.shape != times.shape: + base_result['note'] = 'Skipped; exposure times were not aligned for fast UltraNest binning.' + return base_result + + point_count = int(times.shape[0]) + if point_count <= int(min_points_to_bin): + base_result['note'] = ( + f"Skipped; {point_count} point(s) did not exceed the fast UltraNest " + f"binning threshold of {int(min_points_to_bin)}." + ) + return base_result + + max_points = int(max(1, max_points)) + target_points = min(max_points, point_count) + valid = ( + np.isfinite(times) + & np.isfinite(flux_values) + & np.isfinite(flux_errors) + & (flux_errors > 0) + & np.isfinite(airmass) + ) + if jd_array is not None: + valid &= np.isfinite(jd_array) + if exposure_array is not None: + valid &= np.isfinite(exposure_array) + if np.count_nonzero(valid) <= target_points: + base_result['note'] = 'Skipped; too few finite points remained for fast UltraNest binning.' + return base_result + + ordered_indices = np.flatnonzero(valid)[np.argsort(times[valid])] + chunks = [chunk for chunk in np.array_split(ordered_indices, target_points) if chunk.size > 0] + if len(chunks) >= point_count or not chunks: + base_result['note'] = 'Skipped; fast UltraNest binning would not reduce the light curve.' + return base_result + + binned_time = [] + binned_flux = [] + binned_unc = [] + binned_airmass = [] + binned_jd = [] if jd_array is not None else None + binned_exposure = [] if exposure_array is not None else None + for chunk in chunks: + chunk_unc = flux_errors[chunk] + weights = np.zeros(chunk_unc.shape, dtype=float) + valid_unc = np.isfinite(chunk_unc) & (chunk_unc > 0) + weights[valid_unc] = 1.0 / (chunk_unc[valid_unc] ** 2) + binned_time.append(_weighted_mean_with_fallback(times[chunk], weights)) + binned_flux.append(_weighted_mean_with_fallback(flux_values[chunk], weights)) + if np.any(weights > 0): + binned_unc.append(float(np.sqrt(1.0 / np.sum(weights[weights > 0])))) + else: + scatter = float(np.nanstd(flux_values[chunk])) + binned_unc.append(scatter / np.sqrt(max(chunk.size, 1)) if np.isfinite(scatter) else np.nan) + binned_airmass.append(_weighted_mean_with_fallback(airmass[chunk], weights)) + if jd_array is not None: + binned_jd.append(_weighted_mean_with_fallback(jd_array[chunk], weights)) + if exposure_array is not None: + binned_exposure.append(_weighted_mean_with_fallback(exposure_array[chunk], weights)) + + binned_time = np.asarray(binned_time, dtype=float) + binned_flux = np.asarray(binned_flux, dtype=float) + binned_unc = np.asarray(binned_unc, dtype=float) + binned_airmass = np.asarray(binned_airmass, dtype=float) + finite_binned = ( + np.isfinite(binned_time) + & np.isfinite(binned_flux) + & np.isfinite(binned_unc) + & (binned_unc > 0) + & np.isfinite(binned_airmass) + ) + if binned_jd is not None: + binned_jd = np.asarray(binned_jd, dtype=float) + finite_binned &= np.isfinite(binned_jd) + if binned_exposure is not None: + binned_exposure = np.asarray(binned_exposure, dtype=float) + finite_binned &= np.isfinite(binned_exposure) + + if np.count_nonzero(finite_binned) < LIGHTCURVE_MIN_VALID_POINTS: + base_result['note'] = 'Skipped; fast UltraNest binning produced too few finite bins.' + return base_result + + result = dict(base_result) + result.update({ + 'applied': True, + 'time': binned_time[finite_binned], + 'flux': binned_flux[finite_binned], + 'unc': binned_unc[finite_binned], + 'airmass': binned_airmass[finite_binned], + 'jd_times': None if binned_jd is None else binned_jd[finite_binned], + 'exposure_times_seconds': None if binned_exposure is None else binned_exposure[finite_binned], + 'binned_point_count': int(np.count_nonzero(finite_binned)), + 'bin_indices': [chunk.tolist() for i, chunk in enumerate(chunks) if finite_binned[i]], + 'note': ( + f"Using fast UltraNest binning for pre-final runs: " + f"{point_count} point(s) -> {int(np.count_nonzero(finite_binned))} binned point(s)." + ), + }) + return result + + +def annotate_fast_ultranest_binning(fit, binning_result): + if fit is None or not isinstance(binning_result, dict): + return + fit.fast_ultranest_binning_applied = bool(binning_result.get('applied', False)) + fit.fast_ultranest_original_point_count = int(binning_result.get('original_point_count', 0)) + fit.fast_ultranest_binned_point_count = int(binning_result.get('binned_point_count', 0)) + fit.fast_ultranest_binning_note = binning_result.get('note') + + +def summarize_initial_fit_transit_coverage( + times, + fit, + flux_values=None, + flux_errors=None, + depth_fraction=OUT_OF_TRANSIT_BASELINE_DEPTH_FRACTION, +): + times = np.asarray(times, dtype=float) + transit_model = np.asarray(getattr(fit, 'transit', []), dtype=float) + if flux_values is None: + flux_values = np.ones_like(times, dtype=float) + else: + flux_values = np.asarray(flux_values, dtype=float) + + if transit_model.shape != times.shape or flux_values.shape != times.shape: + return { + 'valid': False, + 'note': 'initial fit did not provide a transit model aligned with the light curve.', + } + + valid = ( + np.isfinite(times) + & np.isfinite(flux_values) + & (flux_values > 0) + & np.isfinite(transit_model) + ) + if flux_errors is not None: + flux_errors = np.asarray(flux_errors, dtype=float) + if flux_errors.shape == flux_values.shape: + valid &= np.isfinite(flux_errors) & (flux_errors > 0) + + if np.count_nonzero(valid) < 3: + return { + 'valid': False, + 'note': 'not enough finite flux points remain to isolate the modeled transit window.', + } + + depth = np.clip(1.0 - transit_model, 0.0, None) + max_depth = np.nanmax(depth[valid]) + if not np.isfinite(max_depth) or max_depth <= 0: + return { + 'valid': False, + 'note': 'initial fit did not produce a measurable transit depth for baseline isolation.', + } + + threshold = max(1e-6, depth_fraction * max_depth) + in_transit = valid & (depth > threshold) + if not np.any(in_transit): + return { + 'valid': False, + 'note': 'could not isolate ingress and egress from the initial fit.', + } + + ingress_time = float(np.nanmin(times[in_transit])) + egress_time = float(np.nanmax(times[in_transit])) + oot_mask = valid & ((times < ingress_time) | (times > egress_time)) + + mid_transit = float(getattr(fit, 'parameters', {}).get('tmid', np.nanmedian(times[valid]))) + pre_mask = oot_mask & (times < mid_transit) + post_mask = oot_mask & (times > mid_transit) + pre_points = int(np.count_nonzero(pre_mask)) + post_points = int(np.count_nonzero(post_mask)) + has_two_sided_oot = pre_points > 0 and post_points > 0 + + summary = { + 'valid': True, + 'note': None, + 'mid_transit': mid_transit, + 'ingress_time': ingress_time, + 'egress_time': egress_time, + 'in_transit_mask': in_transit, + 'oot_mask': oot_mask, + 'pre_mask': pre_mask, + 'post_mask': post_mask, + 'pre_points': pre_points, + 'post_points': post_points, + 'has_two_sided_oot': has_two_sided_oot, + } + if not has_two_sided_oot: + summary['note'] = 'need out-of-transit coverage on both sides of transit to fit a linear baseline.' + return summary + + +def summarize_prior_transit_coverage( + times, + prior, + flux_values=None, + flux_errors=None, +): + times = np.asarray(times, dtype=float) + if flux_values is None: + flux_values = np.ones_like(times, dtype=float) + else: + flux_values = np.asarray(flux_values, dtype=float) + + if times.shape != flux_values.shape: + return { + 'valid': False, + 'note': 'prior-based transit coverage could not be aligned with the light curve.', + } + + valid = np.isfinite(times) & np.isfinite(flux_values) & (flux_values > 0) + if flux_errors is not None: + flux_errors = np.asarray(flux_errors, dtype=float) + if flux_errors.shape == flux_values.shape: + valid &= np.isfinite(flux_errors) & (flux_errors > 0) + + if np.count_nonzero(valid) < 3: + return { + 'valid': False, + 'note': 'not enough finite flux points remain to evaluate prior-based transit coverage.', + } + + try: + mid_transit = float(prior.get('tmid', np.nan)) + except (AttributeError, TypeError, ValueError): + mid_transit = np.nan + if not np.isfinite(mid_transit): + return { + 'valid': False, + 'note': 'prior-based transit coverage skipped: no finite ephemeris-centered Tmid was available.', + } + + duration = estimate_transit_duration_from_prior_geometry(prior) + if not np.isfinite(duration) or duration <= 0: + return { + 'valid': False, + 'note': 'prior-based transit coverage skipped: could not estimate a physical transit duration from the priors.', + } + + ingress_time = float(mid_transit - 0.5 * duration) + egress_time = float(mid_transit + 0.5 * duration) + in_transit = valid & (times >= ingress_time) & (times <= egress_time) + if not np.any(in_transit): + return { + 'valid': False, + 'note': 'prior-based transit coverage skipped: the ephemeris-centered transit window does not overlap the observations.', + } + + oot_mask = valid & ((times < ingress_time) | (times > egress_time)) + pre_mask = oot_mask & (times < mid_transit) + post_mask = oot_mask & (times > mid_transit) + pre_points = int(np.count_nonzero(pre_mask)) + post_points = int(np.count_nonzero(post_mask)) + has_two_sided_oot = pre_points > 0 and post_points > 0 + + summary = { + 'valid': True, + 'note': None, + 'mid_transit': mid_transit, + 'ingress_time': ingress_time, + 'egress_time': egress_time, + 'in_transit_mask': in_transit, + 'oot_mask': oot_mask, + 'pre_mask': pre_mask, + 'post_mask': post_mask, + 'pre_points': pre_points, + 'post_points': post_points, + 'has_two_sided_oot': has_two_sided_oot, + 'used_prior_ephemeris': True, + 'duration': duration, + } + if not has_two_sided_oot: + summary['note'] = ( + 'need out-of-transit coverage on both sides of the ephemeris-centered transit window ' + 'to fit a linear baseline.' + ) + return summary + + +def _coverage_duration_from_context(prior, duration_prior=None): + if isinstance(duration_prior, dict): + duration = coerce_finite_transit_qc_scalar(duration_prior.get('expected_duration', np.nan)) + if np.isfinite(duration) and duration > 0: + return float(duration) + return estimate_transit_duration_from_prior_geometry(prior) + + +def _coverage_tmid_from_context(prior, tmid_search_summary=None): + if isinstance(tmid_search_summary, dict): + tmid = coerce_finite_transit_qc_scalar(tmid_search_summary.get('tmid', np.nan)) + if np.isfinite(tmid): + return float(tmid) + try: + return float(prior.get('tmid', np.nan)) + except (AttributeError, TypeError, ValueError): + return np.nan + + +def build_ephemeris_tmid_search_summary_for_coverage( + times, + planet_dict, + prior=None, + duration_prior=None, + sigma_multiplier=35.0, +): + if not isinstance(planet_dict, dict): + return None + prior = prior if isinstance(prior, dict) else {} + + prior_tmid = coerce_finite_transit_qc_scalar( + planet_dict.get('midT', prior.get('tmid', np.nan)) + ) + period = coerce_finite_transit_qc_scalar( + planet_dict.get('pPer', prior.get('per', np.nan)) + ) + midt_unc = coerce_finite_transit_qc_scalar(planet_dict.get('midTUnc', 0.0)) + per_unc = coerce_finite_transit_qc_scalar(planet_dict.get('pPerUnc', 0.0)) + if not np.isfinite(midt_unc): + midt_unc = 0.0 + if not np.isfinite(per_unc): + per_unc = 0.0 + if not np.isfinite(prior_tmid) or not np.isfinite(period) or period <= 0: + return None + + coverage_prior = dict(prior) + coverage_prior.setdefault('tmid', prior_tmid) + coverage_prior.setdefault('per', period) + coverage_prior.setdefault('rprs', planet_dict.get('rprs', np.nan)) + coverage_prior.setdefault('ars', planet_dict.get('aRs', np.nan)) + coverage_prior.setdefault('inc', planet_dict.get('inc', np.nan)) + coverage_prior.setdefault('ecc', planet_dict.get('ecc', 0.0)) + coverage_prior.setdefault('omega', planet_dict.get('omega', 0.0)) + expected_duration = _coverage_duration_from_context( + coverage_prior, + duration_prior=duration_prior, + ) + + return estimate_ephemeris_tmid_and_bounds( + times, + prior_tmid, + period, + midt_unc, + per_unc, + expected_duration=expected_duration, + sigma_multiplier=sigma_multiplier, + ) + + +def expected_transit_observed_segment( + observed_start, + observed_end, + ingress_time, + mid_transit, + egress_time, +): + if observed_end < ingress_time: + return "pre-transit baseline only" + if observed_start > egress_time: + return "post-transit baseline only" + + pieces = [] + if observed_start < ingress_time: + pieces.append("pre-ingress baseline") + if observed_start <= ingress_time <= observed_end: + pieces.append("ingress") + if observed_start <= mid_transit <= observed_end: + pieces.append("mid-transit") + if observed_start <= egress_time <= observed_end: + pieces.append("egress") + if observed_end > egress_time: + pieces.append("post-egress baseline") + if not pieces: + if observed_end < mid_transit: + return "inside the first half of transit" + if observed_start > mid_transit: + return "inside the second half of transit" + return "inside the expected transit" + return " plus ".join(pieces) + + +def score_expected_transit_model_success( + transit_fraction_observed, + covers_ingress, + covers_mid_transit, + covers_egress, + pre_points, + post_points, +): + has_two_sided_baseline = pre_points > 0 and post_points > 0 + if transit_fraction_observed <= 0: + return "very low", 0.05 + if transit_fraction_observed < 0.25: + return "very low", 0.15 + if not has_two_sided_baseline: + if transit_fraction_observed >= 0.9 and covers_ingress and covers_egress: + return "moderate", 0.50 + if transit_fraction_observed >= 0.5 and covers_mid_transit: + return "low", 0.35 + return "low", 0.25 + if transit_fraction_observed >= 0.9 and covers_ingress and covers_egress: + return "high", 0.85 + if transit_fraction_observed >= 0.65 and covers_mid_transit and (covers_ingress or covers_egress): + return "moderate", 0.65 + if transit_fraction_observed >= 0.4: + return "low", 0.40 + return "low", 0.25 + + +def build_expected_transit_coverage_assessment( + times, + prior, + flux_values=None, + flux_errors=None, + tmid_search_summary=None, + duration_prior=None, +): + times = np.asarray(times, dtype=float) + if flux_values is None: + flux_values = np.ones_like(times, dtype=float) + else: + flux_values = np.asarray(flux_values, dtype=float) + + base = { + 'valid': False, + 'point_count': 0, + 'observed_start': np.nan, + 'observed_end': np.nan, + 'observed_span': np.nan, + 'expected_tmid': np.nan, + 'expected_duration': np.nan, + 'expected_ingress_time': np.nan, + 'expected_egress_time': np.nan, + 'overlap_duration': 0.0, + 'transit_fraction_observed': 0.0, + 'pre_ingress_points': 0, + 'in_transit_points': 0, + 'post_egress_points': 0, + 'covers_ingress': False, + 'covers_mid_transit': False, + 'covers_egress': False, + 'observed_segment': 'unknown', + 'success_label': 'unknown', + 'success_chance': np.nan, + 'expected_successful': False, + 'note': 'Could not evaluate expected transit coverage before UltraNest.', + } + + if times.shape != flux_values.shape: + base['note'] = 'Could not evaluate expected transit coverage because time and flux arrays were misaligned.' + return base + + valid = np.isfinite(times) & np.isfinite(flux_values) & (flux_values > 0) + if flux_errors is not None: + flux_errors = np.asarray(flux_errors, dtype=float) + if flux_errors.shape == flux_values.shape: + valid &= np.isfinite(flux_errors) & (flux_errors > 0) + + if np.count_nonzero(valid) < 3: + base['note'] = 'Could not evaluate expected transit coverage because too few finite light-curve points remain.' + return base + + finite_times = np.sort(times[valid]) + observed_start = float(finite_times[0]) + observed_end = float(finite_times[-1]) + observed_span = float(observed_end - observed_start) + mid_transit = _coverage_tmid_from_context(prior, tmid_search_summary=tmid_search_summary) + duration = _coverage_duration_from_context(prior, duration_prior=duration_prior) + base.update({ + 'point_count': int(finite_times.size), + 'observed_start': observed_start, + 'observed_end': observed_end, + 'observed_span': observed_span, + 'expected_tmid': mid_transit, + 'expected_duration': duration, + }) + + if not np.isfinite(mid_transit): + base['note'] = 'Could not evaluate expected transit coverage because no finite ephemeris Tmid was available.' + return base + if not np.isfinite(duration) or duration <= 0: + base['note'] = 'Could not evaluate expected transit coverage because the expected transit duration is unavailable.' + return base + + ingress_time = float(mid_transit - 0.5 * duration) + egress_time = float(mid_transit + 0.5 * duration) + in_transit_mask = valid & (times >= ingress_time) & (times <= egress_time) + pre_mask = valid & (times < ingress_time) + post_mask = valid & (times > egress_time) + overlap_start = max(observed_start, ingress_time) + overlap_end = min(observed_end, egress_time) + overlap_duration = max(0.0, float(overlap_end - overlap_start)) + transit_fraction_observed = float(np.clip(overlap_duration / duration, 0.0, 1.0)) + covers_ingress = observed_start <= ingress_time <= observed_end + covers_mid_transit = observed_start <= mid_transit <= observed_end + covers_egress = observed_start <= egress_time <= observed_end + observed_segment = expected_transit_observed_segment( + observed_start, + observed_end, + ingress_time, + mid_transit, + egress_time, + ) + success_label, success_chance = score_expected_transit_model_success( + transit_fraction_observed, + covers_ingress, + covers_mid_transit, + covers_egress, + int(np.count_nonzero(pre_mask)), + int(np.count_nonzero(post_mask)), + ) + expected_successful = success_chance >= 0.5 + + if expected_successful: + note = ( + "The observed timestamps appear to contain enough of the expected transit window " + "for a constrained nested fit." + ) + elif transit_fraction_observed <= 0: + note = ( + "The observed timestamps do not overlap the expected transit window; " + "UltraNest is unlikely to recover a constrained transit solution." + ) + elif int(np.count_nonzero(pre_mask)) == 0 or int(np.count_nonzero(post_mask)) == 0: + note = ( + "The expected transit is not bracketed by out-of-transit data on both sides; " + "UltraNest may chase partial-transit or baseline-degenerate solutions." + ) + else: + note = ( + "The expected transit is only partially observed; UltraNest may return broad or " + "edge-hugging posteriors." + ) + + base.update({ + 'valid': True, + 'expected_ingress_time': ingress_time, + 'expected_egress_time': egress_time, + 'overlap_duration': overlap_duration, + 'transit_fraction_observed': transit_fraction_observed, + 'pre_ingress_points': int(np.count_nonzero(pre_mask)), + 'in_transit_points': int(np.count_nonzero(in_transit_mask)), + 'post_egress_points': int(np.count_nonzero(post_mask)), + 'covers_ingress': bool(covers_ingress), + 'covers_mid_transit': bool(covers_mid_transit), + 'covers_egress': bool(covers_egress), + 'observed_segment': observed_segment, + 'success_label': success_label, + 'success_chance': float(success_chance), + 'expected_successful': bool(expected_successful), + 'note': note, + }) + return base + + +def _format_minutes_from_days(days): + try: + value = float(days) * 24.0 * 60.0 + except (TypeError, ValueError): + return "n/a" + return "n/a" if not np.isfinite(value) else f"{value:.1f} min" + + +def log_expected_transit_coverage_assessment(assessment, indent=" "): + if not isinstance(assessment, dict): + return + + if not assessment.get('valid'): + log_info( + f"{indent}Warning: pre-UltraNest transit coverage assessment unavailable: " + f"{assessment.get('note', 'unknown reason')}", + warn=True, + ) + return + + success_label = str(assessment.get('success_label', 'unknown')).upper() + success_chance = coerce_finite_transit_qc_scalar(assessment.get('success_chance', np.nan)) + success_text = success_label + if np.isfinite(success_chance): + success_text = f"{success_label} (~{100.0 * float(success_chance):.0f}%)" + + warn = not bool(assessment.get('expected_successful', False)) + log_info(f"{indent}Pre-UltraNest transit coverage assessment:", warn=warn) + log_info( + f"{indent} Data time range: {assessment['observed_start']:.8f} to " + f"{assessment['observed_end']:.8f} BJD_TDB " + f"({_format_minutes_from_days(assessment.get('observed_span'))}, " + f"{assessment.get('point_count', 0)} point(s)).", + warn=warn, + ) + log_info( + f"{indent} Expected transit window: ingress {assessment['expected_ingress_time']:.8f}, " + f"mid {assessment['expected_tmid']:.8f}, egress {assessment['expected_egress_time']:.8f} " + f"BJD_TDB (duration {_format_minutes_from_days(assessment.get('expected_duration'))}).", + warn=warn, + ) + log_info( + f"{indent} Observed coverage: {assessment.get('observed_segment', 'unknown')}; " + f"{100.0 * assessment.get('transit_fraction_observed', 0.0):.1f}% of the expected transit " + f"window with {assessment.get('pre_ingress_points', 0)} pre-ingress, " + f"{assessment.get('in_transit_points', 0)} in-transit, and " + f"{assessment.get('post_egress_points', 0)} post-egress point(s).", + warn=warn, + ) + log_info( + f"{indent} Estimated fit success: {success_text}. {assessment.get('note', '')}", + warn=warn, + ) + + +def extract_baseline_corrected_lightcurve_arrays(fit): + times = np.asarray(getattr(fit, 'time', []), dtype=float) + if times.ndim != 1 or times.size == 0: + return None, None, None + + flux_values = None + flux_source = None + + detrended = np.asarray(getattr(fit, 'detrended', []), dtype=float) + if detrended.shape == times.shape: + valid_detrended = np.isfinite(detrended) & (detrended > 0) + if np.any(valid_detrended): + flux_values = detrended.copy() + flux_source = "current detrended light curve" + + if flux_values is None: + data = np.asarray(getattr(fit, 'data', []), dtype=float) + airmass_model = np.asarray(getattr(fit, 'airmass_model', []), dtype=float) + if data.shape == times.shape and airmass_model.shape == times.shape: + with np.errstate(divide='ignore', invalid='ignore'): + flux_values = np.divide(data, airmass_model) + flux_source = "current flux ratio divided by the fitted airmass/baseline model" + elif data.shape == times.shape: + flux_values = data.copy() + flux_source = "current raw flux ratio" + else: + return None, None, None + + flux_errors = None + detrended_errors = np.asarray(getattr(fit, 'detrendederr', []), dtype=float) + if detrended_errors.shape == times.shape: + valid_detrended_errors = np.isfinite(detrended_errors) & (detrended_errors > 0) + if np.any(valid_detrended_errors): + flux_errors = detrended_errors.copy() + + if flux_errors is None: + data_errors = np.asarray(getattr(fit, 'dataerr', []), dtype=float) + airmass_model = np.asarray(getattr(fit, 'airmass_model', []), dtype=float) + if data_errors.shape == times.shape and airmass_model.shape == times.shape: + with np.errstate(divide='ignore', invalid='ignore'): + flux_errors = np.divide(data_errors, airmass_model) + elif data_errors.shape == times.shape: + flux_errors = data_errors.copy() + else: + flux_errors = np.full(times.shape, np.nan, dtype=float) + + return flux_values, flux_errors, flux_source + + +def prepare_final_fit_lightcurve_series( + fit, + depth_fraction=OUT_OF_TRANSIT_BASELINE_DEPTH_FRACTION, +): + times = np.asarray(getattr(fit, 'time', []), dtype=float) + if times.ndim != 1 or times.size == 0: + return { + 'applied': False, + 'note': 'could not prepare a final-fit light curve because the current fit had no time samples.', + } + + flux_values, flux_errors, flux_source = extract_baseline_corrected_lightcurve_arrays(fit) + if flux_values is None: + return { + 'applied': False, + 'note': 'could not derive a baseline-corrected light curve for final-fit preparation from the current fit.', + } + + flux_values = np.asarray(flux_values, dtype=float) + flux_errors = np.asarray(flux_errors, dtype=float) + + valid_flux = np.isfinite(flux_values) & (flux_values > 0) + valid_errors = np.isfinite(flux_errors) & (flux_errors > 0) + + if np.count_nonzero(valid_flux) < LIGHTCURVE_MIN_VALID_POINTS: + return { + 'applied': False, + 'note': 'not enough finite baseline-corrected flux points remained for final-fit preparation.', + } + + coverage_summary = summarize_initial_fit_transit_coverage( + times, + fit, + flux_values=flux_values, + flux_errors=flux_errors if np.any(valid_errors) else None, + depth_fraction=depth_fraction, + ) + + baseline_mask = valid_flux + used_two_sided_oot = False + pre_points = coverage_summary.get('pre_points', 0) + post_points = coverage_summary.get('post_points', 0) + + if coverage_summary.get('valid') and coverage_summary.get('has_two_sided_oot'): + candidate_baseline_mask = coverage_summary['oot_mask'] & valid_flux + if np.count_nonzero(candidate_baseline_mask) >= LIGHTCURVE_MIN_VALID_POINTS: + baseline_mask = candidate_baseline_mask + used_two_sided_oot = True + + baseline_level, baseline_scatter = sigma_clipped_nanmedian(flux_values[baseline_mask]) + if not np.isfinite(baseline_level) or baseline_level <= 0: + fallback_mask = valid_flux + baseline_level, baseline_scatter = sigma_clipped_nanmedian(flux_values[fallback_mask]) + baseline_mask = fallback_mask + + if not np.isfinite(baseline_level) or baseline_level <= 0: + return { + 'applied': False, + 'note': 'could not determine a positive baseline level for final-fit preparation.', + } + + normalized_flux = flux_values / baseline_level + normalized_unc = flux_errors / baseline_level + + if used_two_sided_oot: + uncertainty_mask = baseline_mask & np.isfinite(normalized_unc) & (normalized_unc > 0) + observed_scatter = np.nanstd(normalized_flux[baseline_mask]) + predicted_unc = np.nanmedian(normalized_unc[uncertainty_mask]) if np.any(uncertainty_mask) else np.nan + if np.isfinite(observed_scatter) and observed_scatter > 0 and np.isfinite(predicted_unc) and predicted_unc > 0: + normalized_unc *= observed_scatter / predicted_unc + + valid_normalized_unc = np.isfinite(normalized_unc) & (normalized_unc > 0) + if not np.any(valid_normalized_unc): + fallback_unc = baseline_scatter / baseline_level + if not np.isfinite(fallback_unc) or fallback_unc <= 0: + fallback_unc = np.nanstd(normalized_flux[baseline_mask]) + if not np.isfinite(fallback_unc) or fallback_unc <= 0: + fallback_unc = np.finfo(float).eps + normalized_unc = np.full(times.shape, fallback_unc, dtype=float) + else: + fallback_unc = np.nanmedian(normalized_unc[valid_normalized_unc]) + if not np.isfinite(fallback_unc) or fallback_unc <= 0: + fallback_unc = np.nanstd(normalized_flux[baseline_mask]) + if not np.isfinite(fallback_unc) or fallback_unc <= 0: + fallback_unc = np.finfo(float).eps + normalized_unc[~valid_normalized_unc] = fallback_unc + + if used_two_sided_oot: + note = ( + f"Prepared the final-fit input light curve from the {flux_source} and normalized it with " + f"{pre_points} pre-ingress and {post_points} post-egress modeled out-of-transit point(s)." + ) + else: + note = ( + f"Prepared the final-fit input light curve from the {flux_source} and normalized it with a " + f"sigma-clipped full-series baseline because the current fit only bracketed one side of transit " + f"({pre_points} pre-ingress and {post_points} post-egress modeled out-of-transit point(s))." + ) + + return { + 'applied': True, + 'flux': normalized_flux, + 'unc': normalized_unc, + 'note': note, + 'source': flux_source, + 'coverage_summary': coverage_summary, + 'used_two_sided_oot': used_two_sided_oot, + 'baseline_level': float(baseline_level), + } + + +def normalize_out_of_transit_baseline_min_side_points(value): + if value is None: + return 0 + try: + return max(0, int(value)) + except (TypeError, ValueError): + return OUT_OF_TRANSIT_BASELINE_MIN_SIDE_POINTS_DEFAULT + + +def fit_airmass_baseline_parameters_on_out_of_transit( + times, + flux_values, + flux_errors, + airmass, + fit, + prior=None, + bounds=None, + depth_fraction=OUT_OF_TRANSIT_BASELINE_DEPTH_FRACTION, + min_side_points=OUT_OF_TRANSIT_BASELINE_MIN_SIDE_POINTS_DEFAULT, +): + times = np.asarray(times, dtype=float) + flux_values = np.asarray(flux_values, dtype=float) + flux_errors = np.asarray(flux_errors, dtype=float) + airmass = np.asarray(airmass, dtype=float) + prior = {} if prior is None else dict(prior) + bounds = {} if bounds is None else dict(bounds) + + base_result = { + 'applied': False, + 'note': 'out-of-transit baseline parameter fitting did not run.', + 'oot_mask': None, + 'pre_points': 0, + 'post_points': 0, + 'a0': np.nan, + 'a0_error': np.nan, + 'a2': prior.get('a2', 0.0), + 'a2_error': np.nan, + 'used_prior_ephemeris': False, + } + + if not (times.shape == flux_values.shape == flux_errors.shape == airmass.shape): + base_result['note'] = 'light-curve arrays could not be aligned for out-of-transit baseline fitting.' + return base_result + + coverage_summary = summarize_initial_fit_transit_coverage( + times, + fit, + flux_values=flux_values, + flux_errors=flux_errors, + depth_fraction=depth_fraction, + ) + if not coverage_summary.get('valid') and prior: + prior_coverage = summarize_prior_transit_coverage( + times, + prior, + flux_values=flux_values, + flux_errors=flux_errors, + ) + if prior_coverage.get('valid'): + coverage_summary = prior_coverage + base_result['used_prior_ephemeris'] = True + + if not coverage_summary.get('valid'): + base_result['note'] = coverage_summary.get( + 'note', + 'could not isolate out-of-transit points for baseline parameter fitting.', + ) + return base_result + + oot_mask = np.asarray(coverage_summary.get('oot_mask'), dtype=bool) + finite_mask = ( + oot_mask + & np.isfinite(times) + & np.isfinite(flux_values) + & (flux_values > 0) + & np.isfinite(flux_errors) + & (flux_errors > 0) + & np.isfinite(airmass) + ) + point_count = int(np.count_nonzero(finite_mask)) + fit_a2 = 'a2' in bounds + min_points = 3 if fit_a2 else 2 + base_result['pre_points'] = coverage_summary.get('pre_points', 0) + base_result['post_points'] = coverage_summary.get('post_points', 0) + min_side_points = normalize_out_of_transit_baseline_min_side_points(min_side_points) + if ( + min_side_points > 0 + and ( + base_result['pre_points'] <= min_side_points + or base_result['post_points'] <= min_side_points + ) + ): + base_result['note'] = ( + f"only {base_result['pre_points']} pre-ingress and " + f"{base_result['post_points']} post-egress out-of-transit point(s) were available; " + f"need more than {min_side_points} on each side to fit baseline parameters." + ) + return base_result + + if point_count < min_points: + base_result['note'] = ( + f"only {point_count} finite out-of-transit point(s) were available; " + f"need at least {min_points} to fit baseline parameters." + ) + return base_result + + reference_airmass = transit_qc_airmass_reference(airmass) + x = airmass[finite_mask] - reference_airmass + y = flux_values[finite_mask] + yerr = flux_errors[finite_mask] + + a0_bounds = bounds.get('a0') or bounds.get('a1') or [0.5, 1.5] + try: + a0_lower, a0_upper = np.asarray(a0_bounds, dtype=float).reshape(-1)[:2] + except (TypeError, ValueError, IndexError): + a0_lower, a0_upper = 0.5, 1.5 + if not np.isfinite(a0_lower) or a0_lower <= 0: + a0_lower = max(np.nanmedian(y) * 0.5, np.finfo(float).eps) + if not np.isfinite(a0_upper) or a0_upper <= a0_lower: + a0_upper = max(np.nanmedian(y) * 1.5, a0_lower * 1.01) + + if fit_a2: + try: + a2_lower, a2_upper = np.asarray(bounds.get('a2'), dtype=float).reshape(-1)[:2] + except (TypeError, ValueError, IndexError): + a2_lower, a2_upper = -3.0, 3.0 + if not np.isfinite(a2_lower) or not np.isfinite(a2_upper) or a2_lower >= a2_upper: + a2_lower, a2_upper = -3.0, 3.0 + else: + a2_lower = a2_upper = float(prior.get('a2', 0.0) or 0.0) + + initial_a0 = float(np.clip(np.nanmedian(y), a0_lower, a0_upper)) + initial_a2 = float(prior.get('a2', 0.0) or 0.0) + if fit_a2: + initial_a2 = float(np.clip(initial_a2, a2_lower, a2_upper)) + + if fit_a2: + initial = np.array([np.log(initial_a0), initial_a2], dtype=float) + lower_bounds = np.array([np.log(a0_lower), a2_lower], dtype=float) + upper_bounds = np.array([np.log(a0_upper), a2_upper], dtype=float) + else: + initial = np.array([np.log(initial_a0)], dtype=float) + lower_bounds = np.array([np.log(a0_lower)], dtype=float) + upper_bounds = np.array([np.log(a0_upper)], dtype=float) + + def residuals(params): + log_a0 = params[0] + a2_value = params[1] if fit_a2 else initial_a2 + model = np.exp(log_a0) * np.exp(a2_value * x) + return (y - model) / yerr + + try: + result = least_squares( + residuals, + x0=initial, + bounds=(lower_bounds, upper_bounds), + jac='3-point', + loss='linear', + ) + except (ValueError, np.linalg.LinAlgError): + base_result['note'] = 'weighted out-of-transit baseline parameter fit failed.' + return base_result + + if not getattr(result, 'success', False) or not np.all(np.isfinite(result.x)): + base_result['note'] = 'weighted out-of-transit baseline parameter fit did not converge.' + return base_result + + log_a0 = float(result.x[0]) + a0 = float(np.exp(log_a0)) + a2 = float(result.x[1] if fit_a2 else initial_a2) + jacobian = np.asarray(result.jac, dtype=float) + residual_vector = np.asarray(result.fun, dtype=float) + dof = max(1, residual_vector.size - result.x.size) + reduced_chi2 = np.sum(residual_vector ** 2) / dof + covariance = None + if jacobian.ndim == 2 and jacobian.shape[0] >= jacobian.shape[1]: + try: + covariance = np.linalg.pinv(jacobian.T @ jacobian) + covariance *= max(float(reduced_chi2), 1.0) + except np.linalg.LinAlgError: + covariance = None + + if covariance is not None and covariance.shape[0] >= 1: + log_a0_error = float(np.sqrt(max(covariance[0, 0], 0.0))) + a0_error = abs(a0) * log_a0_error + else: + a0_error = np.nan + if covariance is not None and fit_a2 and covariance.shape[0] >= 2: + a2_error = float(np.sqrt(max(covariance[1, 1], 0.0))) + else: + a2_error = ( + estimate_fixed_airmass_coefficient_error(y, yerr, airmass[finite_mask]) + if not fit_a2 else np.nan + ) + if a2_error is None: + a2_error = 0.0 + + if not np.isfinite(a0_error) or a0_error <= 0: + a0_error = float(np.nanmedian(yerr)) + if fit_a2 and (not np.isfinite(a2_error) or a2_error <= 0): + airmass_span_value = np.nanmax(x) - np.nanmin(x) + if np.isfinite(airmass_span_value) and airmass_span_value > 0: + a2_error = float(np.nanmedian(yerr / np.maximum(y, np.finfo(float).eps)) / airmass_span_value) + else: + a2_error = 0.0 + + side_note = ( + f"{coverage_summary.get('pre_points', 0)} pre-ingress and " + f"{coverage_summary.get('post_points', 0)} post-egress out-of-transit point(s)" + ) + if base_result['used_prior_ephemeris']: + side_note += " from the ephemeris-centered transit window" + + return { + 'applied': True, + 'note': ( + "Fitted a0" + + (" and a2" if fit_a2 else "") + + f" using only {side_note}; these baseline terms are fixed/profiled in the final transit fit." + ), + 'oot_mask': finite_mask, + 'pre_points': coverage_summary.get('pre_points', 0), + 'post_points': coverage_summary.get('post_points', 0), + 'a0': a0, + 'a0_error': float(a0_error), + 'a2': a2, + 'a2_error': float(a2_error), + 'used_prior_ephemeris': base_result['used_prior_ephemeris'], + } + + +def detrend_flux_on_out_of_transit_baseline( + times, + flux_values, + flux_errors, + fit, + prior=None, + depth_fraction=OUT_OF_TRANSIT_BASELINE_DEPTH_FRACTION, + min_side_points=OUT_OF_TRANSIT_BASELINE_MIN_SIDE_POINTS_DEFAULT, +): + times = np.asarray(times, dtype=float) + flux_values = np.asarray(flux_values, dtype=float) + flux_errors = np.asarray(flux_errors, dtype=float) + if flux_errors.shape != flux_values.shape: + flux_errors = np.ones_like(flux_values, dtype=float) + + coverage_summary = summarize_initial_fit_transit_coverage( + times, + fit, + flux_values=flux_values, + flux_errors=flux_errors, + depth_fraction=depth_fraction, + ) + used_prior_coverage = False + if prior is not None and ( + (not coverage_summary.get('valid')) + or (not coverage_summary.get('has_two_sided_oot')) + ): + prior_coverage_summary = summarize_prior_transit_coverage( + times, + prior, + flux_values=flux_values, + flux_errors=flux_errors, + ) + if prior_coverage_summary.get('valid') and prior_coverage_summary.get('has_two_sided_oot'): + coverage_summary = prior_coverage_summary + used_prior_coverage = True + + if not coverage_summary.get('valid'): + return { + 'applied': False, + 'note': coverage_summary.get('note', 'could not isolate a transit window for baseline fitting.'), + } + + if not coverage_summary.get('has_two_sided_oot'): + return { + 'applied': False, + 'note': coverage_summary.get( + 'note', + 'need out-of-transit coverage on both sides of transit to fit a linear baseline.', + ), + 'pre_points': coverage_summary.get('pre_points', 0), + 'post_points': coverage_summary.get('post_points', 0), + } + + oot_mask = coverage_summary['oot_mask'] + mid_transit = coverage_summary['mid_transit'] + ingress_time = coverage_summary['ingress_time'] + egress_time = coverage_summary['egress_time'] + pre_points = coverage_summary['pre_points'] + post_points = coverage_summary['post_points'] + min_side_points = normalize_out_of_transit_baseline_min_side_points(min_side_points) + if min_side_points > 0 and (pre_points <= min_side_points or post_points <= min_side_points): + return { + 'applied': False, + 'note': ( + f"only {pre_points} pre-ingress and {post_points} post-egress out-of-transit point(s) " + f"were available; need more than {min_side_points} on each side to fit a linear baseline." + ), + 'pre_points': pre_points, + 'post_points': post_points, + } + + x = times[oot_mask] - mid_transit + if np.allclose(x, x[0]): + return { + 'applied': False, + 'note': 'out-of-transit timestamps do not span enough time to fit a line.', + 'pre_points': pre_points, + 'post_points': post_points, + } + + design = np.column_stack((np.ones_like(x), x)) + oot_errors = flux_errors[oot_mask] + weights = np.ones_like(x, dtype=float) + valid_weights = np.isfinite(oot_errors) & (oot_errors > 0) + if np.any(valid_weights): + weights = np.zeros_like(x, dtype=float) + weights[valid_weights] = 1.0 / (oot_errors[valid_weights] ** 2) + if not np.any(weights > 0): + weights = np.ones_like(x, dtype=float) + + sqrt_weights = np.sqrt(weights) + try: + coeffs, _, _, _ = np.linalg.lstsq(design * sqrt_weights[:, None], flux_values[oot_mask] * sqrt_weights, rcond=None) + except np.linalg.LinAlgError: + return { + 'applied': False, + 'note': 'linear out-of-transit baseline fit failed.', + 'pre_points': pre_points, + 'post_points': post_points, + } + + intercept, slope = coeffs + baseline = intercept + slope * (times - mid_transit) + if not np.all(np.isfinite(baseline)) or np.any(baseline <= 0): + return { + 'applied': False, + 'note': 'linear baseline prediction became non-physical for part of the light curve.', + 'pre_points': pre_points, + 'post_points': post_points, + } + + return { + 'applied': True, + 'note': ( + ( + "Applied weighted linear out-of-transit baseline detrending using the " + "ephemeris-centered transit window from the priors because the fitted transit " + "window was one-sided. " + if used_prior_coverage else + "Applied weighted linear out-of-transit baseline detrending using " + ) + + f"{pre_points} pre-ingress and {post_points} post-egress points." + ), + 'flux': flux_values / baseline, + 'unc': flux_errors / baseline, + 'baseline': baseline, + 'slope': float(slope), + 'intercept': float(intercept), + 'reference_time_bjd_tdb': float(mid_transit), + 'pre_points': pre_points, + 'post_points': post_points, + 'ingress_time': ingress_time, + 'egress_time': egress_time, + 'used_prior_ephemeris': used_prior_coverage, + } + + +def estimate_transit_duration_from_fit(fit): + if fit is None: + return np.nan + + for attribute_name in ('duration_expected', 'duration_measured'): + duration = getattr(fit, attribute_name, np.nan) + if np.isfinite(duration) and duration > 0: + return float(duration) + + times = np.asarray(getattr(fit, 'time', []), dtype=float) + transit_model = np.asarray(getattr(fit, 'transit', []), dtype=float) + if times.shape != transit_model.shape or times.size == 0: + return np.nan + + in_transit = np.isfinite(times) & np.isfinite(transit_model) & (transit_model < 1) + if not np.any(in_transit): + return np.nan + + transit_times = np.sort(times[in_transit]) + if transit_times.size == 1: + sorted_times = np.sort(times[np.isfinite(times)]) + if sorted_times.size < 2: + return np.nan + cadence = np.nanmedian(np.diff(sorted_times)) + return float(cadence) if np.isfinite(cadence) and cadence > 0 else np.nan + + cadence = np.nanmedian(np.diff(np.sort(times[np.isfinite(times)]))) + if not np.isfinite(cadence) or cadence <= 0: + cadence = 0.0 + duration = (transit_times[-1] - transit_times[0]) + cadence + return float(duration) if np.isfinite(duration) and duration > 0 else np.nan + + +def build_nested_tmid_refinement_from_initial_fit(times, flux_values, flux_errors, prior, bounds, fit): + original_tmid_bounds = clone_lightcurve_bounds(bounds).get('tmid') + base_plan = { + 'applied': False, + 'note': 'Not needed; using the original nested-sampling Tmid bounds.', + 'prior': dict(prior), + 'bounds': clone_lightcurve_bounds(bounds), + 'original_tmid_bounds': original_tmid_bounds, + 'refined_tmid_bounds': original_tmid_bounds, + } + + if fit is None or not hasattr(fit, 'parameters') or not isinstance(fit.parameters, dict): + base_plan['note'] = "Skipped; the initial LM fit did not provide fitted parameters for nested-sampling refinement." + return base_plan + + tmid = fit.parameters.get('tmid', np.nan) + if not np.isfinite(tmid): + base_plan['note'] = "Skipped; the initial LM fit did not return a finite Tmid." + return base_plan + + coverage_summary = summarize_initial_fit_transit_coverage( + times, + fit, + flux_values=flux_values, + flux_errors=flux_errors, + ) + if not coverage_summary.get('valid'): + base_plan['note'] = ( + "Skipped; the initial LM fit did not provide a usable modeled transit window for nested-sampling refinement." + ) + return base_plan + + if not coverage_summary.get('has_two_sided_oot'): + pre_points = coverage_summary.get('pre_points', 0) + post_points = coverage_summary.get('post_points', 0) + base_plan['note'] = ( + "Skipped; the initial LM fit only captured one side of the modeled transit, " + "so tightening the nested-sampling Tmid bounds would lock onto a partial-transit solution " + f"({pre_points} pre-ingress and {post_points} post-egress out-of-transit point(s))." + ) + return base_plan + + duration = estimate_transit_duration_from_fit(fit) + period = fit.parameters.get('per', prior.get('per', np.nan)) + if not np.isfinite(duration) or duration <= 0: + base_plan['note'] = "Skipped; the initial LM fit did not produce a measurable transit duration." + return base_plan + if np.isfinite(period) and duration >= period: + base_plan['note'] = "Skipped; the initial LM fit returned a non-physical transit duration." + return base_plan + + valid_times = np.asarray(times, dtype=float) + valid_times = valid_times[np.isfinite(valid_times)] + cadence = np.nan + if valid_times.size > 1: + cadence = np.nanmedian(np.diff(np.sort(valid_times))) + + half_width = FINAL_FIT_TMID_HALF_DURATION_MULTIPLIER * duration + if np.isfinite(cadence) and cadence > 0: + half_width = max(half_width, 3.0 * cadence) + if not np.isfinite(half_width) or half_width <= 0: + base_plan['note'] = "Skipped; the nested-sampling Tmid refinement half-width was not physical." + return base_plan + + refined_lower = float(tmid - half_width) + refined_upper = float(tmid + half_width) + if original_tmid_bounds is not None and len(original_tmid_bounds) == 2: + original_lower = float(original_tmid_bounds[0]) + original_upper = float(original_tmid_bounds[1]) + refined_lower = max(original_lower, refined_lower) + refined_upper = min(original_upper, refined_upper) + + if not np.isfinite(refined_lower) or not np.isfinite(refined_upper) or refined_upper <= refined_lower: + base_plan['note'] = "Skipped; the refined nested-sampling Tmid bounds collapsed to an invalid range." + return base_plan + + refined_tmid_bounds = [refined_lower, refined_upper] + base_plan['refined_tmid_bounds'] = refined_tmid_bounds + if original_tmid_bounds is not None and np.allclose( + np.asarray(original_tmid_bounds, dtype=float), + np.asarray(refined_tmid_bounds, dtype=float), + atol=1e-12, + rtol=0.0, + ): + base_plan['note'] = ( + "Not needed; the initial LM fit already sat inside the original nested-sampling Tmid bounds." + ) + return base_plan + + refined_prior = dict(prior) + for key in ('rprs', 'ars', 'tmid', 'inc', 'a2'): + if key in refined_prior and key in fit.parameters: + refined_prior[key] = fit.parameters[key] + + refined_bounds = clone_lightcurve_bounds(bounds) + refined_bounds['tmid'] = refined_tmid_bounds + + base_plan.update({ + 'applied': True, + 'note': ( + "Using the initial LM fit to recenter nested-sampling Tmid bounds to " + f"[{refined_lower:.6f}, {refined_upper:.6f}] around Tmid={tmid:.6f}." + ), + 'prior': refined_prior, + 'bounds': refined_bounds, + }) + return base_plan + + +def build_final_fit_prefit_refinement_plan( + times, + flux_values, + flux_errors, + airmass, + prior, + bounds, + fit, + jd_times=None, + exposure_times_seconds=None, + baseline_duration_multiplier=FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT, +): + times = np.asarray(times, dtype=float) + flux_values = np.asarray(flux_values, dtype=float) + flux_errors = np.asarray(flux_errors, dtype=float) + airmass = np.asarray(airmass, dtype=float) + jd_array = None if jd_times is None else np.asarray(jd_times, dtype=float) + exposure_array = None if exposure_times_seconds is None else np.asarray(exposure_times_seconds, dtype=float) + if exposure_array is not None and exposure_array.shape != times.shape: + exposure_array = None + + original_tmid_bounds = clone_lightcurve_bounds(bounds).get('tmid') + base_plan = { + 'applied': False, + 'note': None, + 'baseline_duration_multiplier': float(baseline_duration_multiplier), + 'duration': np.nan, + 'trimmed_pre_points': 0, + 'trimmed_post_points': 0, + 'original_point_count': int(times.shape[0]), + 'refined_point_count': int(times.shape[0]), + 'original_tmid_bounds': original_tmid_bounds, + 'refined_tmid_bounds': original_tmid_bounds, + 'times': times, + 'flux': flux_values, + 'unc': flux_errors, + 'airmass': airmass, + 'jd_times': jd_array, + 'exposure_times_seconds': exposure_array, + 'prior': dict(prior), + 'bounds': clone_lightcurve_bounds(bounds), + } + + if fit is None: + base_plan['note'] = "Skipped; the initial nested fit did not return a solution." + return base_plan + + duration = estimate_transit_duration_from_fit(fit) + base_plan['duration'] = duration + if not np.isfinite(duration) or duration <= 0: + base_plan['note'] = "Skipped; the initial nested fit did not produce a measurable transit duration." + return base_plan + + fit_parameters = getattr(fit, 'parameters', {}) + tmid = fit_parameters.get('tmid', prior.get('tmid', np.nan)) + if not np.isfinite(tmid): + base_plan['note'] = "Skipped; the initial nested fit did not return a finite Tmid." + return base_plan + + coverage_summary = summarize_initial_fit_transit_coverage( + times, + fit, + flux_values=flux_values, + flux_errors=flux_errors, + ) + if coverage_summary.get('valid') and not coverage_summary.get('has_two_sided_oot'): + pre_points = coverage_summary.get('pre_points', 0) + post_points = coverage_summary.get('post_points', 0) + base_plan['note'] = ( + "Skipped; the initial nested fit only captured one side of the modeled transit, " + "so tightening the second-pass Tmid bounds would lock onto a partial-transit solution " + f"({pre_points} pre-ingress and {post_points} post-egress out-of-transit point(s))." + ) + return base_plan + + period = fit_parameters.get('per', prior.get('per', np.nan)) + if np.isfinite(period) and duration >= period: + base_plan['note'] = "Skipped; the initial nested fit returned a non-physical transit duration." + return base_plan + + half_duration = FINAL_FIT_TMID_HALF_DURATION_MULTIPLIER * duration + if not np.isfinite(half_duration) or half_duration <= 0: + base_plan['note'] = "Skipped; the initial nested fit returned an invalid transit duration." + return base_plan + + keep_half_width = duration * (0.5 + baseline_duration_multiplier) + lower_window = tmid - keep_half_width + upper_window = tmid + keep_half_width + keep_mask = np.isfinite(times) & (times >= lower_window) & (times <= upper_window) + + trimmed_pre_points = int(np.count_nonzero(np.isfinite(times) & (times < lower_window))) + trimmed_post_points = int(np.count_nonzero(np.isfinite(times) & (times > upper_window))) + kept_points = int(np.count_nonzero(keep_mask)) + min_required_points = max(LIGHTCURVE_MIN_VALID_POINTS, len(bounds) + 1) + if kept_points < min_required_points: + keep_mask = np.ones(times.shape[0], dtype=bool) + kept_points = int(keep_mask.sum()) + trimmed_pre_points = 0 + trimmed_post_points = 0 + trim_note = ( + "kept the full light curve because trimming would leave too few points for a stable refit" + ) + elif trimmed_pre_points or trimmed_post_points: + trim_note = ( + f"trimmed {trimmed_pre_points} pre-ingress and {trimmed_post_points} post-egress point(s)" + ) + else: + trim_note = "kept the full light curve because no extra baseline points fell outside the target window" + + refined_lower = float(tmid - half_duration) + refined_upper = float(tmid + half_duration) + if original_tmid_bounds is not None and len(original_tmid_bounds) == 2: + original_lower = float(original_tmid_bounds[0]) + original_upper = float(original_tmid_bounds[1]) + refined_lower = max(original_lower, refined_lower) + refined_upper = min(original_upper, refined_upper) + + if not np.isfinite(refined_lower) or not np.isfinite(refined_upper) or refined_upper <= refined_lower: + base_plan['note'] = "Skipped; the refined final-fit Tmid bounds collapsed to an invalid range." + return base_plan + + refined_tmid_bounds = [refined_lower, refined_upper] + base_plan['refined_tmid_bounds'] = refined_tmid_bounds + + bounds_changed = False + if original_tmid_bounds is None: + bounds_changed = True + else: + bounds_changed = not np.allclose( + np.asarray(original_tmid_bounds, dtype=float), + np.asarray(refined_tmid_bounds, dtype=float), + atol=1e-12, + rtol=0.0, + ) + + if not np.any(keep_mask): + base_plan['note'] = "Skipped; no valid points remained inside the requested prefit window." + return base_plan + + refined_times = times[keep_mask] + refined_flux = flux_values[keep_mask] + refined_unc = flux_errors[keep_mask] + refined_airmass = airmass[keep_mask] + refined_jd_times = None if jd_array is None else jd_array[keep_mask] + refined_exposure_times = None if exposure_array is None else exposure_array[keep_mask] + + refined_prior = dict(prior) + if isinstance(fit_parameters, dict): + for key in ('rprs', 'ars', 'tmid', 'inc', 'a2'): + if key in refined_prior and key in fit_parameters: + refined_prior[key] = fit_parameters[key] + + refined_bounds = clone_lightcurve_bounds(bounds) + refined_bounds['tmid'] = refined_tmid_bounds + + base_plan.update({ + 'applied': bool(trimmed_pre_points or trimmed_post_points or bounds_changed), + 'note': ( + "Using an initial nested fit to estimate a transit duration of " + f"{duration:.6f} day(s), {trim_note}, and setting the second-pass " + f"Tmid bounds to [{refined_tmid_bounds[0]:.6f}, {refined_tmid_bounds[1]:.6f}]." + ), + 'trimmed_pre_points': trimmed_pre_points, + 'trimmed_post_points': trimmed_post_points, + 'refined_point_count': kept_points, + 'times': refined_times, + 'flux': refined_flux, + 'unc': refined_unc, + 'airmass': refined_airmass, + 'jd_times': refined_jd_times, + 'exposure_times_seconds': refined_exposure_times, + 'prior': refined_prior, + 'bounds': refined_bounds, + }) + + if not base_plan['applied']: + base_plan['note'] = ( + "Not needed; the initial final-fit solution already used the desired baseline window " + "and the Tmid bounds already matched the modeled transit duration." + ) + + return base_plan + + +def fit_final_lightcurve_with_oot_baseline_detrending( + times, + flux_values, + flux_errors, + airmass, + prior, + bounds, + jd_times=None, + exposure_times_seconds=None, + skip_airmass_fit=False, + airmass_skip_note=None, + disable_vertical_flux_normalization=False, + detrend_on_outoftransit_baseline=True, + oot_baseline_min_points_per_side=OUT_OF_TRANSIT_BASELINE_MIN_SIDE_POINTS_DEFAULT, + use_impactparameter_rather_than_inclination_to_fit=True, + plot_time_range=None, + baseline_duration_multiplier=FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT, + expected_planet_dict=None, + expected_tmid_search_summary=None, + eebls_search_summary=None, + duration_prior=None, + extend_sparse_posterior_live_points=True, + keep_ultranest_sampler_for_deferred_extension=False, + fix_baseline_terms_for_final=True, + pre_ultranest_coverage_assessment=None, + search_restriction_prior=None, +): + search_restriction_prior = ( + dict(search_restriction_prior) + if isinstance(search_restriction_prior, dict) + else dict(prior) if isinstance(prior, dict) else {} + ) + exposure_times_array = None if exposure_times_seconds is None else np.asarray(exposure_times_seconds, dtype=float) + if exposure_times_array is not None and exposure_times_array.shape != np.asarray(times).shape: + exposure_times_array = None + search_restriction_prior = enrich_search_restriction_prior_with_rprs_data_uncertainty( + search_restriction_prior, + times, + flux_values, + flux_errors, + prior, + context_label="final light curve", + ) + bounds = widen_rprs_bounds_to_data_uncertainty_window(bounds, search_restriction_prior) + if duration_prior is None and expected_planet_dict is not None: + duration_prior = build_single_transit_duration_prior(expected_planet_dict) + if pre_ultranest_coverage_assessment is None: + pre_ultranest_coverage_assessment = build_expected_transit_coverage_assessment( + times, + prior, + flux_values=flux_values, + flux_errors=flux_errors, + tmid_search_summary=expected_tmid_search_summary, + duration_prior=duration_prior, + ) + log_expected_transit_coverage_assessment(pre_ultranest_coverage_assessment) + sparse_posterior_live_point_extension_enabled = should_use_sparse_posterior_live_point_retry( + os.environ.get( + SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_ENV, + SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_DEFAULT, + ) + ) + keep_ultranest_for_sparse_extension = ( + sparse_posterior_live_point_extension_enabled + and ( + extend_sparse_posterior_live_points + or keep_ultranest_sampler_for_deferred_extension + ) + ) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + times, + flux_values, + flux_errors, + airmass, + prior, + bounds, + jd_times=jd_times, + exposure_times_seconds=exposure_times_array, + use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + duration_prior=duration_prior, + keep_ultranest_sampler=keep_ultranest_for_sparse_extension, + pre_ultranest_coverage_assessment=pre_ultranest_coverage_assessment, + search_restriction_prior=search_restriction_prior, + ) + fit = apply_plot_time_range(fit, times if plot_time_range is None else plot_time_range) + annotate_airmass_fit(fit, airmass, skip_airmass_fit, note=airmass_skip_note) + annotate_transit_qc_fit_context( + fit, + planet_dict=expected_planet_dict, + tmid_search_summary=expected_tmid_search_summary, + eebls_search_summary=eebls_search_summary, + ) + annotate_pre_ultranest_transit_coverage(fit, pre_ultranest_coverage_assessment) + + effective_bounds = get_posterior_refit_final_bounds(fit, bounds) + prefit_plan = build_final_fit_prefit_refinement_plan( + times, + flux_values, + flux_errors, + airmass, + prior, + effective_bounds, + fit, + jd_times=jd_times, + exposure_times_seconds=exposure_times_array, + baseline_duration_multiplier=baseline_duration_multiplier, + ) + working_times = prefit_plan['times'] + working_flux = prefit_plan['flux'] + working_unc = prefit_plan['unc'] + working_airmass = prefit_plan['airmass'] + working_jd_times = prefit_plan['jd_times'] + working_exposure_times = prefit_plan.get('exposure_times_seconds') + working_prior = prefit_plan['prior'] + working_bounds = prefit_plan['bounds'] + + if prefit_plan.get('applied'): + log_info("Applying final-fit prefit refinement before the optional baseline detrending pass.") + log_info(prefit_plan['note']) + apply_vertical_flux_normalization_bound( + working_prior, + working_bounds, + working_flux, + disable_vertical_flux_normalization, + ) + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + working_times, + working_flux, + working_unc, + working_airmass, + working_prior, + working_bounds, + jd_times=working_jd_times, + exposure_times_seconds=working_exposure_times, + use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + duration_prior=duration_prior, + keep_ultranest_sampler=keep_ultranest_for_sparse_extension, + pre_ultranest_coverage_assessment=pre_ultranest_coverage_assessment, + search_restriction_prior=search_restriction_prior, + ) + fit = apply_plot_time_range(fit, working_times if plot_time_range is None else plot_time_range) + annotate_airmass_fit(fit, working_airmass, skip_airmass_fit, note=airmass_skip_note) + annotate_transit_qc_fit_context( + fit, + planet_dict=expected_planet_dict, + tmid_search_summary=expected_tmid_search_summary, + eebls_search_summary=eebls_search_summary, + ) + annotate_pre_ultranest_transit_coverage(fit, pre_ultranest_coverage_assessment) + + working_bounds = get_posterior_refit_final_bounds(fit, working_bounds) + + annotate_final_fit_prefit_refinement( + fit, + prefit_plan.get('applied', False), + note=prefit_plan.get('note'), + baseline_duration_multiplier=baseline_duration_multiplier, + duration=prefit_plan.get('duration'), + original_point_count=prefit_plan.get('original_point_count'), + refined_point_count=prefit_plan.get('refined_point_count'), + trimmed_pre_points=prefit_plan.get('trimmed_pre_points', 0), + trimmed_post_points=prefit_plan.get('trimmed_post_points', 0), + original_tmid_bounds=prefit_plan.get('original_tmid_bounds'), + refined_tmid_bounds=prefit_plan.get('refined_tmid_bounds'), + ) + + if detrend_on_outoftransit_baseline: + baseline_parameter_result = fit_airmass_baseline_parameters_on_out_of_transit( + working_times, + working_flux, + working_unc, + working_airmass, + fit, + prior=working_prior, + bounds=working_bounds, + min_side_points=oot_baseline_min_points_per_side, + ) + else: + baseline_parameter_result = { + 'applied': False, + 'note': 'Disabled with out-of-transit baseline detrending.', + 'pre_points': 0, + 'post_points': 0, + } + if baseline_parameter_result.get('applied') and not fix_baseline_terms_for_final: + baseline_parameter_result = { + 'applied': False, + 'note': 'Deferred; pre-final UltraNest runs keep a0 and a2 as simultaneous fitted parameters.', + 'pre_points': baseline_parameter_result.get('pre_points', 0), + 'post_points': baseline_parameter_result.get('post_points', 0), + } + baseline_fit_mask = None + baseline_fixed_errors = {} + baseline_constrained_prior = dict(working_prior) + baseline_constrained_bounds = clone_lightcurve_bounds(working_bounds) + pre_detrending_baseline = None + if baseline_parameter_result.get('applied'): + log_info("Prepared out-of-transit airmass/baseline parameter constraints for a fallback final transit refit.") + log_info(baseline_parameter_result['note']) + baseline_fit_mask = np.asarray(baseline_parameter_result['oot_mask'], dtype=bool) + baseline_fixed_errors = baseline_fixed_errors_from_oot_parameter_result(baseline_parameter_result) + baseline_constrained_prior['a0'] = baseline_parameter_result['a0'] + baseline_constrained_prior['a1'] = baseline_parameter_result['a0'] + baseline_constrained_prior['a2'] = baseline_parameter_result['a2'] + for key in ('rprs', 'ars', 'tmid', 'inc'): + if key in baseline_constrained_prior and key in getattr(fit, 'parameters', {}): + baseline_constrained_prior[key] = fit.parameters[key] + baseline_constrained_bounds.pop('a0', None) + baseline_constrained_bounds.pop('a1', None) + baseline_constrained_bounds.pop('a2', None) + pre_detrending_baseline = { + 'source': "out-of-transit airmass/baseline parameter fit before linear baseline detrending", + 'scale_parameter': 'a0', + 'scale_value': baseline_parameter_result.get('a0'), + 'scale_error': baseline_parameter_result.get('a0_error'), + 'a2_value': baseline_parameter_result.get('a2'), + 'a2_error': baseline_parameter_result.get('a2_error'), + } + else: + annotate_out_of_transit_baseline_parameter_fit( + fit, + False, + note=baseline_parameter_result.get('note'), + pre_points=baseline_parameter_result.get('pre_points', 0), + post_points=baseline_parameter_result.get('post_points', 0), + ) + + def run_oot_baseline_parameter_refit_if_needed(current_fit): + if not baseline_parameter_result.get('applied'): + return current_fit + + refit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + working_times, + working_flux, + working_unc, + working_airmass, + baseline_constrained_prior, + baseline_constrained_bounds, + jd_times=working_jd_times, + exposure_times_seconds=working_exposure_times, + use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + duration_prior=duration_prior, + keep_ultranest_sampler=keep_ultranest_for_sparse_extension, + baseline_fit_mask=baseline_fit_mask, + fixed_parameter_errors=baseline_fixed_errors, + fixed_flux_baseline=True, + pre_ultranest_coverage_assessment=pre_ultranest_coverage_assessment, + search_restriction_prior=search_restriction_prior, + ) + refit = apply_plot_time_range(refit, working_times if plot_time_range is None else plot_time_range) + annotate_airmass_fit(refit, working_airmass, skip_airmass_fit, note=airmass_skip_note) + annotate_transit_qc_fit_context( + refit, + planet_dict=expected_planet_dict, + tmid_search_summary=expected_tmid_search_summary, + eebls_search_summary=eebls_search_summary, + ) + annotate_pre_ultranest_transit_coverage(refit, pre_ultranest_coverage_assessment) + annotate_final_fit_prefit_refinement( + refit, + prefit_plan.get('applied', False), + note=prefit_plan.get('note'), + baseline_duration_multiplier=baseline_duration_multiplier, + duration=prefit_plan.get('duration'), + original_point_count=prefit_plan.get('original_point_count'), + refined_point_count=prefit_plan.get('refined_point_count'), + trimmed_pre_points=prefit_plan.get('trimmed_pre_points', 0), + trimmed_post_points=prefit_plan.get('trimmed_post_points', 0), + original_tmid_bounds=prefit_plan.get('original_tmid_bounds'), + refined_tmid_bounds=prefit_plan.get('refined_tmid_bounds'), + ) + annotate_out_of_transit_baseline_parameter_fit( + refit, + True, + note=baseline_parameter_result.get('note'), + pre_points=baseline_parameter_result.get('pre_points', 0), + post_points=baseline_parameter_result.get('post_points', 0), + a0=baseline_parameter_result.get('a0'), + a0_error=baseline_parameter_result.get('a0_error'), + a2=baseline_parameter_result.get('a2'), + a2_error=baseline_parameter_result.get('a2_error'), + ) + return refit + + if not detrend_on_outoftransit_baseline: + fit = run_oot_baseline_parameter_refit_if_needed(fit) + annotate_out_of_transit_baseline_detrending( + fit, + False, + note="Disabled; using the direct nested-sampling fit.", + ) + annotate_transit_detection_qc(fit) + if extend_sparse_posterior_live_points: + fit = extend_sparse_posterior_live_points_if_needed( + fit, + enabled=sparse_posterior_live_point_extension_enabled, + ) + annotate_transit_detection_qc(fit) + return fit, working_flux, working_unc + + detrend_result = detrend_flux_on_out_of_transit_baseline( + working_times, + working_flux, + working_unc, + fit, + prior=working_prior, + min_side_points=oot_baseline_min_points_per_side, + ) + if not detrend_result.get('applied'): + note = f"Skipped; {detrend_result.get('note', 'unable to fit an out-of-transit baseline.')}" + log_info(f"Optional out-of-transit baseline detrending skipped: {detrend_result.get('note', 'unknown reason')}") + fit = run_oot_baseline_parameter_refit_if_needed(fit) + annotate_out_of_transit_baseline_detrending( + fit, + False, + note=note, + pre_points=detrend_result.get('pre_points', 0), + post_points=detrend_result.get('post_points', 0), + ) + annotate_transit_detection_qc(fit) + if extend_sparse_posterior_live_points: + fit = extend_sparse_posterior_live_points_if_needed( + fit, + enabled=sparse_posterior_live_point_extension_enabled, + ) + annotate_transit_detection_qc(fit) + return fit, working_flux, working_unc + + log_info("Applying optional out-of-transit linear baseline detrending and refitting final light curve.") + log_info(detrend_result['note']) + + refit_prior = dict(working_prior) + for key in ('rprs', 'ars', 'tmid', 'inc'): + if key in refit_prior and key in fit.parameters: + refit_prior[key] = fit.parameters[key] + refit_prior['a0'] = 1.0 + refit_prior['a1'] = 1.0 + refit_prior['a2'] = 0.0 + + refit_bounds = clone_lightcurve_bounds(working_bounds) + for key in ('a0', 'a1', 'a2'): + refit_bounds.pop(key, None) + refit_fixed_parameter_errors = dict(baseline_fixed_errors) + refit_fixed_parameter_errors.setdefault('a0', 0.0) + refit_fixed_parameter_errors.setdefault('a1', refit_fixed_parameter_errors.get('a0', 0.0)) + refit_fixed_parameter_errors['a2'] = 0.0 + baseline_parameter_fit_note = baseline_parameter_result.get('note') + baseline_parameter_fit_used = False + if baseline_parameter_result.get('applied'): + baseline_parameter_fit_note = ( + "Not used in the final refit because the out-of-transit linear detrending " + "already flattened the final-fit flux baseline." + ) + + refit_search_restriction_prior = enrich_search_restriction_prior_with_rprs_data_uncertainty( + search_restriction_prior, + working_times, + detrend_result['flux'], + detrend_result['unc'], + refit_prior, + context_label="out-of-transit detrended final light curve", + ) + refit_bounds = widen_rprs_bounds_to_data_uncertainty_window( + refit_bounds, + refit_search_restriction_prior, + ) + refit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + working_times, + detrend_result['flux'], + detrend_result['unc'], + working_airmass, + refit_prior, + refit_bounds, + jd_times=working_jd_times, + exposure_times_seconds=working_exposure_times, + use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + duration_prior=duration_prior, + keep_ultranest_sampler=keep_ultranest_for_sparse_extension, + baseline_fit_mask=baseline_fit_mask, + fixed_parameter_errors=refit_fixed_parameter_errors, + fixed_flux_baseline=True, + pre_ultranest_coverage_assessment=pre_ultranest_coverage_assessment, + search_restriction_prior=refit_search_restriction_prior, + ) + refit = apply_plot_time_range(refit, working_times if plot_time_range is None else plot_time_range) + annotate_airmass_fit(refit, working_airmass, skip_airmass_fit, note=airmass_skip_note) + annotate_transit_qc_fit_context( + refit, + planet_dict=expected_planet_dict, + tmid_search_summary=expected_tmid_search_summary, + eebls_search_summary=eebls_search_summary, + ) + annotate_pre_ultranest_transit_coverage(refit, pre_ultranest_coverage_assessment) + annotate_final_fit_prefit_refinement( + refit, + prefit_plan.get('applied', False), + note=prefit_plan.get('note'), + baseline_duration_multiplier=baseline_duration_multiplier, + duration=prefit_plan.get('duration'), + original_point_count=prefit_plan.get('original_point_count'), + refined_point_count=prefit_plan.get('refined_point_count'), + trimmed_pre_points=prefit_plan.get('trimmed_pre_points', 0), + trimmed_post_points=prefit_plan.get('trimmed_post_points', 0), + original_tmid_bounds=prefit_plan.get('original_tmid_bounds'), + refined_tmid_bounds=prefit_plan.get('refined_tmid_bounds'), + ) + annotate_out_of_transit_baseline_detrending( + refit, + True, + note=detrend_result['note'], + slope=detrend_result['slope'], + intercept=detrend_result['intercept'], + reference_time_bjd_tdb=detrend_result['reference_time_bjd_tdb'], + pre_points=detrend_result['pre_points'], + post_points=detrend_result['post_points'], + ) + annotate_out_of_transit_baseline_parameter_fit( + refit, + baseline_parameter_fit_used, + note=baseline_parameter_fit_note, + pre_points=baseline_parameter_result.get('pre_points', 0), + post_points=baseline_parameter_result.get('post_points', 0), + a0=baseline_parameter_result.get('a0') if baseline_parameter_result.get('applied') else None, + a0_error=baseline_parameter_result.get('a0_error') if baseline_parameter_result.get('applied') else None, + a2=baseline_parameter_result.get('a2') if baseline_parameter_result.get('applied') else None, + a2_error=baseline_parameter_result.get('a2_error') if baseline_parameter_result.get('applied') else None, + ) + if pre_detrending_baseline is not None: + annotate_pre_detrending_baseline_coefficients(refit, **pre_detrending_baseline) + annotate_transit_detection_qc(refit) + if extend_sparse_posterior_live_points: + refit = extend_sparse_posterior_live_points_if_needed( + refit, + enabled=sparse_posterior_live_point_extension_enabled, + ) + annotate_transit_detection_qc(refit) + return refit, detrend_result['flux'], detrend_result['unc'] + + +def psf_sigma_from_fit(psf_row, fallback_sigma=np.nan): + try: + sigx = float(psf_row[3]) + sigy = float(psf_row[4]) + sigma = 0.5 * (sigx + sigy) + except (IndexError, TypeError, ValueError): + sigma = np.nan + + if np.isfinite(sigma) and sigma > 0: + return float(sigma) + + if np.isfinite(fallback_sigma) and fallback_sigma > 0: + return float(fallback_sigma) + + return np.nan + + +def overexposure_aperture_radius_from_psf_row(psf_row, fallback_sigma=np.nan): + sigma = psf_sigma_from_fit(psf_row, fallback_sigma=fallback_sigma) + if not np.isfinite(sigma) or sigma <= 0: + sigma = 1.0 + return max(float(APERTURE_SIGMA_MAX) * float(sigma), 1.0) + + +def aperture_contains_overexposed_pixel(data, xc, yc, aperture_radius, threshold_value, + fast_mode=False): + if not ( + np.isfinite(xc) + and np.isfinite(yc) + and np.isfinite(aperture_radius) + and aperture_radius > 0 + and np.isfinite(threshold_value) + and threshold_value > 0 + ): + return False + + try: + aperture = CircularAperture(positions=[(float(xc), float(yc))], r=float(aperture_radius)) + mask_method = 'center' if fast_mode else 'exact' + mask = aperture.to_mask(method=mask_method)[0] + data_cutout = mask.cutout(data) + except Exception: + return False + + if data_cutout is None: + return False + + cutout = np.asarray(data_cutout, dtype=float) + weights = np.asarray(mask.data, dtype=float) + valid_pixels = np.isfinite(cutout) & np.isfinite(weights) & (weights > 0) + if not np.any(valid_pixels): + return False + + return bool(np.nanmax(cutout[valid_pixels]) > float(threshold_value)) + + +def representative_psf_sigma(psf_rows, fallback_sigma=np.nan): + try: + sigmas = np.asarray(psf_rows[:, 3], dtype=float) + np.asarray(psf_rows[:, 4], dtype=float) + except (IndexError, TypeError, ValueError): + sigmas = np.array([], dtype=float) + + if sigmas.size: + sigmas *= 0.5 + sigmas[~np.isfinite(sigmas) | (sigmas <= 0)] = np.nan + center, _ = sigma_clipped_nanmedian(sigmas) + if np.isfinite(center) and center > 0: + return float(center) + + if np.isfinite(fallback_sigma) and fallback_sigma > 0: + return float(fallback_sigma) + + return np.nan + + +def summarize_adaptive_aperture_usage(psf_rows, aperture_scale, annulus_scale, fallback_sigma=np.nan): + try: + aperture_scale = float(aperture_scale) + annulus_scale = float(annulus_scale) + except (TypeError, ValueError): + return None + + if not np.isfinite(aperture_scale) or not np.isfinite(annulus_scale): + return None + + rows = np.asarray(psf_rows) + if rows.ndim != 2 or rows.shape[0] == 0: + return None + + frame_sigma = np.array( + [psf_sigma_from_fit(row, fallback_sigma=fallback_sigma) for row in rows], + dtype=float, + ) + frame_sigma[~np.isfinite(frame_sigma) | (frame_sigma <= 0)] = np.nan + + aperture_series = aperture_scale * frame_sigma + annulus_series = annulus_scale * frame_sigma + fwhm_series = GAUSSIAN_SIGMA_TO_FWHM * frame_sigma + + geometry_rows = [ + resolve_sky_annulus_geometry(aperture_radius, annulus_width, psf_sigma=sigma) + if np.isfinite(aperture_radius) and np.isfinite(annulus_width) and np.isfinite(sigma) + else None + for aperture_radius, annulus_width, sigma in zip(aperture_series, annulus_series, frame_sigma) + ] + sky_inner_series = np.array( + [np.nan if geometry is None else geometry['inner_radius'] for geometry in geometry_rows], + dtype=float, + ) + sky_outer_series = np.array( + [np.nan if geometry is None else geometry['outer_radius'] for geometry in geometry_rows], + dtype=float, + ) + sky_pixel_series = np.array( + [np.nan if geometry is None else geometry['effective_sky_pixels'] for geometry in geometry_rows], + dtype=float, + ) + + if not np.any(np.isfinite(aperture_series)) or not np.any(np.isfinite(annulus_series)): + return None + + return { + 'aperture_sigma': aperture_scale, + 'annulus_sigma': annulus_scale, + 'frame_sigma': frame_sigma, + 'fwhm_series': fwhm_series, + 'aperture_series': aperture_series, + 'annulus_series': annulus_series, + 'sky_inner_series': sky_inner_series, + 'sky_outer_series': sky_outer_series, + 'sky_pixel_series': sky_pixel_series, + 'aperture_median': float(np.nanmedian(aperture_series)), + 'aperture_std': float(np.nanstd(aperture_series)), + 'aperture_min': float(np.nanmin(aperture_series)), + 'aperture_max': float(np.nanmax(aperture_series)), + 'annulus_median': float(np.nanmedian(annulus_series)), + 'annulus_std': float(np.nanstd(annulus_series)), + 'annulus_min': float(np.nanmin(annulus_series)), + 'annulus_max': float(np.nanmax(annulus_series)), + } + + +def update_photometry_adaptive_summary(photometry_info, use_adaptive_apertures, aperture_values, annulus_values, + psf_rows, fallback_sigma=np.nan): + photometry_info['adaptive_summary'] = None + + if (not use_adaptive_apertures) or photometry_info.get('min_aperture') in (None, 0): + return None + + a_idx = photometry_info.get('aperture_index') + an_idx = photometry_info.get('annulus_index') + if a_idx is None or an_idx is None or aperture_values is None or annulus_values is None: + return None + + aperture_grid = np.asarray(aperture_values, dtype=float) + annulus_grid = np.asarray(annulus_values, dtype=float) + if a_idx >= aperture_grid.size or an_idx >= annulus_grid.size: + return None + + photometry_info['adaptive_summary'] = summarize_adaptive_aperture_usage( + psf_rows, + aperture_grid[a_idx], + annulus_grid[an_idx], + fallback_sigma=fallback_sigma, + ) + return photometry_info['adaptive_summary'] + + +def reported_photometry_aperture_radii(photometry_info): + adaptive_summary = photometry_info.get('adaptive_summary') + if adaptive_summary is None: + return photometry_info.get('min_aperture'), photometry_info.get('min_annulus') + + aperture = adaptive_summary['aperture_median'] + if photometry_info.get('min_aperture') is not None and photometry_info['min_aperture'] < 0: + aperture = -aperture + return aperture, adaptive_summary['annulus_median'] + + +def build_observing_background_series(psf_data, aper_data, photometry_info, comp_star_count): + use_aperture_background = photometry_info.get('min_aperture') != 0 + a_idx = photometry_info.get('aperture_index') + an_idx = photometry_info.get('annulus_index') + + if use_aperture_background and aper_data is not None and a_idx is not None and an_idx is not None: + background_series = { + 'target': np.asarray(aper_data['target_bg'][:, a_idx, an_idx], dtype=float), + } + for comp_idx in range(comp_star_count): + ckey = f"comp{comp_idx + 1}" + bg_key = f"{ckey}_bg" + if bg_key in aper_data: + background_series[ckey] = np.asarray(aper_data[bg_key][:, a_idx, an_idx], dtype=float) + return background_series + + background_series = { + 'target': np.asarray(psf_data['target'][:, 6], dtype=float), + } + for comp_idx in range(comp_star_count): + ckey = f"comp{comp_idx + 1}" + if ckey in psf_data: + background_series[ckey] = np.asarray(psf_data[ckey][:, 6], dtype=float) + return background_series + + +def resolve_frame_aperture_radii(apertures, annuli, adaptive_apertures=False, frame_sigma=np.nan, + fallback_sigma=np.nan): + aperture_values = np.asarray(apertures, dtype=float).reshape(-1) + annulus_values = np.asarray(annuli, dtype=float).reshape(-1) + + if not adaptive_apertures: + return aperture_values, annulus_values + + sigma_to_use = float(frame_sigma) if np.isfinite(frame_sigma) and frame_sigma > 0 else np.nan + if (not np.isfinite(sigma_to_use) or sigma_to_use <= 0) and np.isfinite(fallback_sigma) and fallback_sigma > 0: + sigma_to_use = float(fallback_sigma) + if not np.isfinite(sigma_to_use) or sigma_to_use <= 0: + sigma_to_use = 1.0 + + return aperture_values * sigma_to_use, annulus_values * sigma_to_use + + +# Initialze plate status log +plateStatus = PlateStatus(log_info) + +def sigma_clip(ogdata, sigma=3, dt=21, po=2, times=None): + values = np.asarray(ogdata, dtype=float) + nanmask = np.isnan(values) + valid_mask = ~nanmask + valid_indices = np.flatnonzero(valid_mask) + + if not (po < dt <= valid_indices.size): + return nanmask + + segment_ranges = [] + if times is not None: + time_values = np.asarray(times, dtype=float) + if time_values.shape == values.shape: + valid_times = time_values[valid_mask] + finite_valid_times = np.isfinite(valid_times) + if np.all(finite_valid_times): + cadence = np.nanmedian(np.diff(valid_times)) if valid_times.size > 1 else np.nan + if np.isfinite(cadence) and cadence > 0: + gap_threshold = max( + 5.0 * cadence, + 0.25 * int(dt) * cadence, + ) + local_start = 0 + for local_index, gap in enumerate(np.diff(valid_times), start=1): + if gap > gap_threshold: + segment_ranges.append((local_start, local_index)) + local_start = local_index + segment_ranges.append((local_start, valid_times.size)) + + if not segment_ranges: + segment_ranges = [(0, valid_indices.size)] + + clipped_mask = np.zeros(valid_indices.size, dtype=bool) + for start, stop in segment_ranges: + local_values = values[valid_indices[start:stop]] + if not (po < dt <= local_values.size): + continue + + mdata = savgol_filter(local_values, window_length=dt, polyorder=po) + # mdata = median_filter(local_values, dt) + res = local_values - mdata + if res.size == 0: + continue + # Vectorized bootstrap estimate avoids Python-loop overhead in tight runs. + sample_size = min(25, res.size) + bootstrap_samples = np.random.choice(res, size=(100, sample_size), replace=True) + std = bn.nanmedian(bn.nanstd(bootstrap_samples, axis=1)) + # std = np.nanstd(res) # biased from large outliers + if not np.isfinite(std) or std <= 0: + continue + sigmask = np.abs(res) > sigma * std + clipped_mask[start:stop] = sigmask + + nanmask[valid_indices] = clipped_mask + return nanmask + + +def adaptive_aperture_outlier_mask(aperture_series, annulus_series=None, sigma=4.5, window=15, polyorder=2): + aperture_series = np.asarray(aperture_series, dtype=float) + combined_mask = _adaptive_series_outlier_mask( + aperture_series, + sigma=sigma, + window=window, + polyorder=polyorder, + ) + + if annulus_series is None: + return combined_mask + + annulus_series = np.asarray(annulus_series, dtype=float) + annulus_mask = _adaptive_series_outlier_mask( + annulus_series, + sigma=sigma, + window=window, + polyorder=polyorder, + ) + return combined_mask | annulus_mask + + +def _adaptive_series_outlier_mask(series, sigma=4.5, window=15, polyorder=2): + values = np.asarray(series, dtype=float) + nanmask = ~np.isfinite(values) + valid_indices = np.flatnonzero(~nanmask) + if valid_indices.size < max(polyorder + 3, 7): + return nanmask + + valid_values = values[valid_indices] + window_length = min(int(window), valid_values.size) + if window_length % 2 == 0: + window_length -= 1 + + if window_length >= polyorder + 2: + trend = savgol_filter(valid_values, window_length=window_length, polyorder=polyorder, mode='interp') + residuals = valid_values - trend + scatter = robust_scatter(residuals) + center = trend + else: + scatter = np.nan + center = np.full(valid_values.shape, bn.nanmedian(valid_values)) + + if not np.isfinite(scatter) or scatter <= 0: + center = np.full(valid_values.shape, bn.nanmedian(valid_values)) + residuals = valid_values - center + scatter = robust_scatter(residuals) + if not np.isfinite(scatter) or scatter <= 0: + return nanmask + + local_mask = np.abs(valid_values - center) > sigma * scatter + outlier_mask = nanmask.copy() + outlier_mask[valid_indices] = local_mask + return outlier_mask + + +def robust_scatter(data): + values = np.asarray(data, dtype=float) + finite = values[np.isfinite(values)] + if finite.size < 2: + return np.nan + + center = bn.nanmedian(finite) + mad = bn.nanmedian(np.abs(finite - center)) + if np.isfinite(mad) and mad > 0: + return 1.4826 * mad + + scatter = bn.nanstd(finite) + if np.isfinite(scatter) and scatter > 0: + return scatter + + return np.nan + + +def expected_transit_depth_from_planet_dict(p_dict): + if not isinstance(p_dict, dict): + return np.nan + + try: + rprs = float(p_dict.get('rprs', np.nan)) + except (TypeError, ValueError): + return np.nan + + if not np.isfinite(rprs) or rprs < 0: + return np.nan + return float(rprs ** 2) + + +def prefit_raw_ratio_outlier_mask( + values, + times=None, + sigma=4.0, + window=11, + min_points=5, + max_iters=2, + expected_transit_depth=None, + min_fractional_deviation=0.05, + use_global=True, +): + values = np.asarray(values, dtype=float).reshape(-1) + outlier_mask = ~np.isfinite(values) | (values <= 0) + valid_indices = np.flatnonzero(~outlier_mask) + if valid_indices.size < max(int(min_points), 3): + return outlier_mask + + if times is not None: + times = np.asarray(times, dtype=float).reshape(-1) + if times.shape == values.shape: + order = np.argsort(times[valid_indices]) + valid_indices = valid_indices[order] + + try: + sigma = float(sigma) + except (TypeError, ValueError): + sigma = 4.0 + if not np.isfinite(sigma) or sigma <= 0: + sigma = 4.0 + + try: + expected_depth = float(expected_transit_depth) + except (TypeError, ValueError): + expected_depth = np.nan + if not np.isfinite(expected_depth) or expected_depth < 0: + expected_depth = 0.0 + + fractional_floor = max(float(min_fractional_deviation), 2.0 * expected_depth) + if not np.isfinite(fractional_floor) or fractional_floor <= 0: + fractional_floor = 0.05 + min_log_deviation = np.log1p(fractional_floor) + + window = max(int(window), 2 * int(min_points) + 1) + if window % 2 == 0: + window += 1 + half_window = window // 2 + min_points = max(int(min_points), 3) + max_iters = max(int(max_iters), 1) + + log_values = np.log(values[valid_indices]) + keep = np.ones(valid_indices.size, dtype=bool) + + for _ in range(max_iters): + newly_rejected = np.zeros(valid_indices.size, dtype=bool) + kept_positions = np.flatnonzero(keep) + if kept_positions.size < min_points: + break + + for position in kept_positions: + lower = max(0, int(position) - half_window) + upper = min(valid_indices.size, int(position) + half_window + 1) + local_positions = np.arange(lower, upper) + local_positions = local_positions[(local_positions != position) & keep[local_positions]] + + if local_positions.size < min_points: + local_positions = kept_positions[kept_positions != position] + if local_positions.size < min_points: + continue + + local_values = log_values[local_positions] + center = bn.nanmedian(local_values) + scatter = robust_scatter(local_values - center) + if not np.isfinite(scatter) or scatter <= 0: + continue + + deviation = abs(log_values[position] - center) + if deviation > sigma * scatter and deviation > min_log_deviation: + newly_rejected[position] = True + + if use_global: + kept_values = log_values[kept_positions] + global_center = bn.nanmedian(kept_values) + global_scatter = robust_scatter(kept_values - global_center) + if np.isfinite(global_scatter) and global_scatter > 0: + global_deviation = np.abs(log_values - global_center) + global_outliers = ( + keep + & (global_deviation > sigma * global_scatter) + & (global_deviation > min_log_deviation) + ) + newly_rejected |= global_outliers + + if not np.any(newly_rejected): + break + keep[newly_rejected] = False + + outlier_mask[valid_indices] = ~keep + return outlier_mask + + +def phase_bin_sigma_clip(values, phase, sigma=3, bins=10, min_points=5, max_iters=3): + values = np.asarray(values, dtype=float) + phase = np.asarray(phase, dtype=float) + nanmask = ~np.isfinite(values) | ~np.isfinite(phase) + valid_indices = np.flatnonzero(~nanmask) + + if valid_indices.size < max(min_points, 3): + return nanmask + + phase_valid = phase[valid_indices] + min_phase = np.nanmin(phase_valid) + max_phase = np.nanmax(phase_valid) + if not np.isfinite(min_phase) or not np.isfinite(max_phase) or min_phase == max_phase: + return nanmask + + bin_count = max(1, int(bins)) + edges = np.linspace(min_phase, max_phase, bin_count + 1) + bin_ids = np.searchsorted(edges[1:-1], phase_valid, side='right') + keep_mask = np.ones(valid_indices.size, dtype=bool) + values_valid = values[valid_indices] + + for bin_id in range(bin_count): + local_positions = np.flatnonzero(bin_ids == bin_id) + if local_positions.size < min_points: + continue + + local_keep = np.ones(local_positions.size, dtype=bool) + for _ in range(max_iters): + candidate_values = values_valid[local_positions][local_keep] + if candidate_values.size < min_points: + break + + center = bn.nanmedian(candidate_values) + scatter = robust_scatter(candidate_values) + if not np.isfinite(scatter) or scatter <= 0: + break + + within_limits = np.abs(candidate_values - center) <= sigma * scatter + if np.all(within_limits): + break + + local_keep[np.flatnonzero(local_keep)[~within_limits]] = False + + keep_mask[local_positions] &= local_keep + + nanmask[valid_indices] = ~keep_mask + return nanmask + + +def apply_lightcurve_mask(lightcurve, mask, sort_index=None): + if lightcurve is None: + return + + mask = np.asarray(mask, dtype=bool) + target_length = mask.shape[0] + if sort_index is not None: + sort_index = np.asarray(sort_index) + target_length = sort_index.shape[0] + + array_attrs = ( + 'time', + 'data', + 'airmass', + 'transit', + 'jd_times', + 'exposure_times_days', + 'phase', + 'residuals', + 'model', + 'detrended', + 'detrendederr', + 'dataerr', + 'airmass_model', + 'wf', + 'stellar_variability_source_indices', + 'stellar_variability_target_flux', + 'stellar_variability_comp_flux', + 'stellar_variability_target_flux_error', + 'stellar_variability_comp_flux_error', + 'stellar_variability_exposure_times_seconds', + ) + + for attr in array_attrs: + if not hasattr(lightcurve, attr): + continue + + values = getattr(lightcurve, attr) + if values is None: + continue + + array_values = np.asarray(values) + if array_values.ndim == 0 or array_values.shape[0] != target_length: + continue + + if sort_index is not None: + array_values = array_values[sort_index] + setattr(lightcurve, attr, array_values[mask]) + + +def apply_plot_time_range(lightcurve, time_values): + if lightcurve is None: + return lightcurve + + values = np.asarray(time_values, dtype=float).reshape(-1) + finite = values[np.isfinite(values)] + if finite.size == 0: + return lightcurve + + lightcurve.plot_time_range = (float(np.min(finite)), float(np.max(finite))) + updater = getattr(lightcurve, "_update_plot_geometry", None) + if callable(updater): + updater() + + return lightcurve + + +def build_time_rejection_diagnostic(stage, times, keep_mask, note=None): + times = np.asarray(times, dtype=float).reshape(-1) + keep_mask = np.asarray(keep_mask, dtype=bool).reshape(-1) + if times.shape[0] != keep_mask.shape[0]: + return None + + finite_mask = np.isfinite(times) + input_point_count = int(np.count_nonzero(finite_mask)) + dropped_times = np.asarray(times[finite_mask & ~keep_mask], dtype=float) + kept_point_count = int(np.count_nonzero(finite_mask & keep_mask)) + dropped_point_count = int(dropped_times.size) + + cadence = np.nan + finite_times = np.sort(times[finite_mask]) + if finite_times.size > 1: + cadence = np.nanmedian(np.diff(finite_times)) + + dropped_ranges = [] + if dropped_times.size: + dropped_times = np.sort(dropped_times) + gap_threshold = np.inf + if np.isfinite(cadence) and cadence > 0: + gap_threshold = max( + TIME_REJECTION_GROUP_GAP_CADENCE_MULTIPLIER * cadence, + np.finfo(float).eps, + ) + + range_start = float(dropped_times[0]) + range_end = float(dropped_times[0]) + range_count = 1 + for current_time in dropped_times[1:]: + current_time = float(current_time) + if np.isfinite(gap_threshold) and (current_time - range_end) <= gap_threshold: + range_end = current_time + range_count += 1 + continue + + dropped_ranges.append({ + 'start': range_start, + 'end': range_end, + 'count': int(range_count), + }) + range_start = current_time + range_end = current_time + range_count = 1 + + dropped_ranges.append({ + 'start': range_start, + 'end': range_end, + 'count': int(range_count), + }) + + return { + 'stage': stage, + 'note': note, + 'input_point_count': input_point_count, + 'kept_point_count': kept_point_count, + 'dropped_point_count': dropped_point_count, + 'cadence': float(cadence) if np.isfinite(cadence) else np.nan, + 'dropped_ranges': dropped_ranges, + 'first_dropped_time': (float(dropped_times[0]) if dropped_times.size else np.nan), + 'last_dropped_time': (float(dropped_times[-1]) if dropped_times.size else np.nan), + } + + +def format_time_rejection_diagnostic(diagnostic, max_ranges=TIME_REJECTION_RANGE_DISPLAY_LIMIT): + if not diagnostic: + return None + + dropped_ranges = diagnostic.get('dropped_ranges') or [] + if not dropped_ranges: + return ( + f"{diagnostic.get('stage', 'frame filter')}: removed 0/" + f"{diagnostic.get('input_point_count', 0)} frame(s)." + ) + + display_ranges = dropped_ranges[:max_ranges] + range_parts = [] + for range_summary in display_ranges: + start = float(range_summary['start']) + end = float(range_summary['end']) + count = int(range_summary['count']) + if count <= 1 or np.isclose(start, end): + range_parts.append(f"{start:.8f} ({count} frame)") + else: + frame_label = "frame" if count == 1 else "frames" + range_parts.append(f"{start:.8f} to {end:.8f} ({count} {frame_label})") + + if len(dropped_ranges) > len(display_ranges): + remaining = len(dropped_ranges) - len(display_ranges) + range_parts.append(f"... {remaining} more range(s)") + + message = ( + f"{diagnostic.get('stage', 'frame filter')}: removed " + f"{diagnostic.get('dropped_point_count', 0)}/{diagnostic.get('input_point_count', 0)} frame(s); " + f"BJD range(s): {'; '.join(range_parts)}." + ) + note = diagnostic.get('note') + if note: + message += f" {note}" + return message + + +def log_lightcurve_filter_diagnostics(diagnostics, header="Lightcurve frame rejection diagnostics", only_removed=True): + normalized = [diagnostic for diagnostic in (diagnostics or []) if diagnostic] + if only_removed: + normalized = [diagnostic for diagnostic in normalized if diagnostic.get('dropped_point_count', 0) > 0] + if not normalized: + return + + log_info(f"\n{header}:") + for diagnostic in normalized: + diagnostic_text = format_time_rejection_diagnostic(diagnostic) + if diagnostic_text: + log_info(f" {diagnostic_text}") + + +EXPOSURE_TIME_HEADER_KEYS = ("EXPTIME", "EFFEXPT", "EXPOSURE", "EXP", "REQTIME", "EXPREQ") +BJD_TDB_MID_EXPOSURE_HEADER_KEYS = ("BJD_TDB", "BJD_TBD", "BJD-TDB", "BJD-MID", "TDB-MID", "BJD", "TDB") +JD_MID_EXPOSURE_HEADER_KEYS = ("JD-MID",) +MJD_MID_EXPOSURE_HEADER_KEYS = ("MJD-MID",) +UTC_MID_EXPOSURE_HEADER_KEYS = ("DATE-AVG", "DATE-MID") +JD_START_EXPOSURE_HEADER_KEYS = ("JD-START", "JD", "JULIAN") +MJD_START_EXPOSURE_HEADER_KEYS = ("MJD-OBS", "MJD") +UTC_START_EXPOSURE_HEADER_KEYS = ("DATE-UTC", "DATE-BEG", "DATE-OBS", "UT-OBS") +UTC_END_EXPOSURE_HEADER_KEYS = ("DATE-END", "END-OBS") +EXPOSURE_VARIATION_REQUIRE_COMP_STAR_FRACTION = 0.01 +HEADER_NUMERIC_TOKEN_RE = re.compile( + r"[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?" +) + + +def header_scalar_value(value): + if isinstance(value, tuple) and value: + return value[0] + return value + + +def finite_header_float(value, allow_unit_text=False): + value = header_scalar_value(value) + if value is None: + return None + if isinstance(value, str) and value.strip().upper() in ("", "UNKNOWN", "N/A", "NA", "NULL", "NONE"): + return None + try: + numeric_value = float(str(value).strip()) + except (TypeError, ValueError): + if not allow_unit_text: + return None + numeric_match = HEADER_NUMERIC_TOKEN_RE.search(str(value)) + if numeric_match is None: + return None + try: + numeric_value = float(numeric_match.group(0)) + except (TypeError, ValueError): + return None + return numeric_value if np.isfinite(numeric_value) else None + + +def first_header_float(hdr, keys, allow_unit_text=False): + for key in keys: + if key not in hdr: + continue + numeric_value = finite_header_float(hdr[key], allow_unit_text=allow_unit_text) + if numeric_value is not None: + return key, numeric_value + return None, None + + +def header_comment_text(hdr, key): + try: + return str(hdr.comments[key]) + except Exception: + return "" + + +def exposure_time_spread_fraction(exptimes): + values = np.asarray(exptimes, dtype=float) + values = values[np.isfinite(values)] + if values.size < 2: + return 0.0 + spread = float(np.nanmax(values) - np.nanmin(values)) + if spread <= 0: + return 0.0 + reference = float(np.nanmedian(values)) + if not np.isfinite(reference) or reference <= 0: + return np.inf + return spread / reference + + +def exposure_variation_requires_comp_star(exptimes, threshold_fraction=EXPOSURE_VARIATION_REQUIRE_COMP_STAR_FRACTION): + spread_fraction = exposure_time_spread_fraction(exptimes) + return (np.isfinite(spread_fraction) and spread_fraction > threshold_fraction) or np.isinf(spread_fraction) + + +def resolve_require_comp_star_for_exposure_times(config_value, exptimes): + require_comp_star = is_comp_star_required(config_value) + if exposure_variation_requires_comp_star(exptimes): + spread_percent = 100.0 * exposure_time_spread_fraction(exptimes) + if not require_comp_star: + log_info( + "Exposure times vary by more than 1% across retained frames " + f"({spread_percent:.2f}%); target-only/no-comparison photometry will scale " + "source counts to a common exposure time before fitting.", + warn=True, + ) + return require_comp_star + + +def parse_header_datetime_to_jd(hdr, key): + if key not in hdr: + return None + + value = header_scalar_value(hdr[key]) + if value is None: + return None + + value_text = str(value).strip() + if not value_text: + return None + + if key == "DATE-OBS" and "T" not in value_text and "TIME-OBS" in hdr: + value_text = f"{value_text}T{header_scalar_value(hdr['TIME-OBS'])}" + elif key == "UT-OBS" and "DATE-OBS" in hdr and "T" not in value_text: + date_text = str(header_scalar_value(hdr["DATE-OBS"])).strip().split("T")[0] + value_text = f"{date_text}T{value_text}" + + try: + dt = dup.parse(value_text) + return Time(dt).jd + except Exception: + return None + + +def first_header_datetime_jd(hdr, keys): + for key in keys: + jd_value = parse_header_datetime_to_jd(hdr, key) + if jd_value is not None and np.isfinite(jd_value): + return key, float(jd_value) + return None, None + + +def direct_bjd_tdb_mid_exposure(hdr): + return first_header_float(hdr, BJD_TDB_MID_EXPOSURE_HEADER_KEYS) + + +def utc_mid_exposure_jd(hdr): + key, jd_mid = first_header_float(hdr, JD_MID_EXPOSURE_HEADER_KEYS) + if jd_mid is not None: + return key, jd_mid + + key, mjd_mid = first_header_float(hdr, MJD_MID_EXPOSURE_HEADER_KEYS) + if mjd_mid is not None: + return key, mjd_mid + 2400000.5 + + return first_header_datetime_jd(hdr, UTC_MID_EXPOSURE_HEADER_KEYS) + + +def utc_start_exposure_jd(hdr): + key, jd_start = first_header_float(hdr, JD_START_EXPOSURE_HEADER_KEYS) + if jd_start is not None: + return key, jd_start + + key, mjd_start = first_header_float(hdr, MJD_START_EXPOSURE_HEADER_KEYS) + if mjd_start is not None: + return key, mjd_start + 2400000.5 + + return first_header_datetime_jd(hdr, UTC_START_EXPOSURE_HEADER_KEYS) + + +def utc_end_exposure_jd(hdr): + return first_header_datetime_jd(hdr, UTC_END_EXPOSURE_HEADER_KEYS) + + +def utc_exposure_midpoint_jd(hdr, exp): + key, jd_mid = utc_mid_exposure_jd(hdr) + if jd_mid is not None: + return key, jd_mid + + start_key, jd_start = utc_start_exposure_jd(hdr) + end_key, jd_end = utc_end_exposure_jd(hdr) + if jd_start is not None and jd_end is not None and jd_end >= jd_start: + return f"{start_key}/{end_key}", 0.5 * (jd_start + jd_end) + + if jd_start is None: + return None, None + + return start_key, jd_start + exp / (2.0 * 60.0 * 60.0 * 24.0) + + +def exp_offset(hdr, time_unit, exp): + """Returns exposure offset (in days) of more than 0 if headers reveals + the time was estimated at the start of the exposure rather than the middle + """ + if 'start' in header_comment_text(hdr, time_unit).lower(): + return exp / (2.0 * 60.0 * 60.0 * 24.0) + return 0.0 + + +def ut_date(hdr, time_unit, exp): + """Converts the Gregorian Date to Julian Date from the header and returns it + along with the exposure offset + """ + if time_unit == 'DATE-OBS': + greg_date = hdr[time_unit] if 'T' in hdr[time_unit] else f"{hdr[time_unit]}T{hdr['TIME-OBS']}" + else: + greg_date = hdr[time_unit] + + dt = dup.parse(greg_date) + atime = Time(dt) + + julian_time = atime.jd + offset = exp_offset(hdr, time_unit, exp) + + return julian_time + offset + + +def julian_date(hdr, time_unit, exp): + """Returns Julian Date from the header along with the exposure offset. + If the image is taken from MicroObservatory (MJD-OBS), + add a timing offset (2400000.5) due to being less precise + """ + time_offset = 2400000.5 if time_unit == 'MJD-OBS' else 0.0 + + julian_time = float(hdr[time_unit]) + time_offset + offset = exp_offset(hdr, time_unit, exp) + + return julian_time + offset + +def get_exp_time(hdr): + _, exp_time = first_header_float(hdr, EXPOSURE_TIME_HEADER_KEYS, allow_unit_text=True) + return exp_time if exp_time is not None else 0.0 + +def img_time_jd(hdr): + """Converts time from the header file to the Julian Date (JD, if needed) + and adds an exposure offset (if needed) + + Parameters + ---------- + hdr : astropy.io.fits.header.Header + A header file that includes the time from when the image was taken + Returns + ------- + float + Time of when the image was taken in the JD with exposure offset + """ + exp = get_exp_time(hdr) + _, jd_mid = utc_exposure_midpoint_jd(hdr, exp) + return jd_mid if jd_mid is not None else np.nan + + +def img_time_bjd_tdb(hdr, p_dict, info_dict): + """Converts time from the header file to BJD-TDB time (if needed) + and adds an exposure offset (if needed) + + Parameters + ---------- + hdr : astropy.io.fits.header.Header + A header file that includes the time from when the image was taken + p_dict: planetary settings dictionary + info_dict: observatory settings dictionary + + Returns + ------- + float + Time of when the image was taken in BJD-TDB with exposure offset + """ + exp = get_exp_time(hdr) + + _, bjd_time = direct_bjd_tdb_mid_exposure(hdr) + if bjd_time is not None: + return bjd_time + + _, jd_time = utc_exposure_midpoint_jd(hdr, exp) + if jd_time is None: + return np.nan + return convert_jd_to_bjd([jd_time], p_dict, info_dict)[0] + +def air_mass(hdr, ra, dec, lat, long, elevation, time): + """Scrapes or calculates the airmass at the time of when the image was taken. + Airmass(X): X = sec(z), z = secant of the zenith angle (angle between zenith and star) + + Parameters + ---------- + hdr : astropy.io.fits.header.Header + A header file that may include the airmass or altitude from when the image was taken + ra : float + Right Ascension + dec : float + Declination + lat : float + Latitude + long : float + Longitude + elevation : float + Elevation/Altitude + + Returns + ------- + float + Airmass value + """ + if 'AIRMASS' in hdr: + am = float(hdr['AIRMASS']) + elif 'TELALT' in hdr: + alt = float(hdr['TELALT']) + cos_am = np.cos((np.pi / 180) * (90.0 - alt)) + am = 1 / cos_am + else: + pointing = SkyCoord(f"{ra} {dec}", unit=(u.deg, u.deg), frame='icrs') + + location = EarthLocation.from_geodetic(lat=lat * u.deg, lon=long * u.deg, height=elevation) + time = Time(time, format='jd', scale='utc', location=location) + point_altaz = pointing.transform_to(AltAz(obstime=time, location=location)) + am = float(point_altaz.secz) + return am + + +def flux_conversion(fluxes, errors, flux_format): + """Converting differential magnitudes to fluxes and calculating its errors + """ + conv = 1000.0 if flux_format == 'millimagnitude' else 1.0 + + pos_err = 10.0 ** (-0.4 * ((fluxes + errors) / conv)) + neg_err = 10.0 ** (-0.4 * ((fluxes - errors) / conv)) + fluxes = 10.0 ** (-0.4 * (fluxes / conv)) + + pos_err_dist = abs(pos_err - fluxes) + neg_err_dist = abs(neg_err - fluxes) + mean_errors = (pos_err_dist * neg_err_dist) ** 0.5 + + return fluxes, mean_errors + + +# Check for difference between NEA and initialization file +def check_parameters(init_parameters, parameters): + different = False + uncert = 1 / 36 + + for key, value in parameters.items(): + if key in ['ra', 'dec'] and init_parameters[key]: + if not parameters[key] - uncert <= init_parameters[key] <= parameters[key] + uncert: + different = True + break + continue + if value != init_parameters[key]: + different = True + break + + if different: + log_info("\nDifference(s) found between initialization file parameters and " + "those scraped by EXOTIC from the NASA Exoplanet Archive." + "\nWould you like:" + "\n (1) EXOTIC to adopt of all of your defined parameters or" + "\n (2) to review the ones scraped from the Archive that differ?") + opt = user_input("Enter 1 or 2: ", type_=int, values=[1, 2]) + + if opt == 2: + return True + else: + return False + + +REQUIRED_TRANSIT_EPHEMERIS_FIELDS = { + 'pPer': { + 'label': 'Orbital Period (days)', + 'uncertainty_key': 'pPerUnc', + }, + 'midT': { + 'label': 'Published Mid-Transit Time (BJD-UTC)', + 'uncertainty_key': 'midTUnc', + }, +} + + +def _positive_finite_ephemeris_value(value): + if isinstance(value, (bool, np.bool_)): + return None + try: + numeric_value = float(value) + except (TypeError, ValueError): + return None + if not np.isfinite(numeric_value) or numeric_value <= 0: + return None + return numeric_value + + +def invalid_required_transit_ephemeris_fields(planet_dict): + if not isinstance(planet_dict, dict): + return list(REQUIRED_TRANSIT_EPHEMERIS_FIELDS) + return [ + key + for key in REQUIRED_TRANSIT_EPHEMERIS_FIELDS + if _positive_finite_ephemeris_value(planet_dict.get(key)) is None + ] + + +def resolve_required_transit_ephemeris(planet_dict, archive_planet_dict=None, archive_lookup=None, + target_name=None): + """Fill missing period/Tmid values from NEA, then fail before reduction if either remains invalid.""" + resolved = dict(planet_dict) if isinstance(planet_dict, dict) else {} + target_name = target_name or resolved.get('pName') + invalid_fields = invalid_required_transit_ephemeris_fields(resolved) + archive_error = None + + if invalid_fields and not isinstance(archive_planet_dict, dict) and callable(archive_lookup): + try: + archive_planet_dict = archive_lookup() + except Exception as exc: + archive_error = exc + + if isinstance(archive_planet_dict, dict): + for key in invalid_fields: + archive_value = _positive_finite_ephemeris_value(archive_planet_dict.get(key)) + if archive_value is None: + continue + + field = REQUIRED_TRANSIT_EPHEMERIS_FIELDS[key] + original_value = resolved.get(key) + resolved[key] = archive_value + log_info( + f"Required ephemeris fallback for {target_name or 'the target'}: " + f"{field['label']} was missing or invalid ({original_value!r}); using NASA Exoplanet " + f"Archive value {archive_value}." + ) + + uncertainty_key = field['uncertainty_key'] + if _positive_finite_ephemeris_value(resolved.get(uncertainty_key)) is None: + archive_uncertainty = _positive_finite_ephemeris_value( + archive_planet_dict.get(uncertainty_key) + ) + if archive_uncertainty is not None: + resolved[uncertainty_key] = archive_uncertainty + + invalid_fields = invalid_required_transit_ephemeris_fields(resolved) + if invalid_fields: + invalid_descriptions = [ + f"{REQUIRED_TRANSIT_EPHEMERIS_FIELDS[key]['label']} ({key})={resolved.get(key)!r}" + for key in invalid_fields + ] + if archive_error is not None: + archive_note = ( + f" NASA Exoplanet Archive fallback failed with " + f"{type(archive_error).__name__}: {archive_error}." + ) + elif isinstance(archive_planet_dict, dict): + archive_note = " The NASA Exoplanet Archive did not provide usable replacement value(s)." + else: + archive_note = " NASA Exoplanet Archive parameters were unavailable." + + message = ( + f"Cannot start EXOTIC reduction for {target_name or 'the target'}: required planetary " + f"ephemeris is missing or invalid: {', '.join(invalid_descriptions)}. Orbital Period and " + f"Published Mid-Transit Time must both be finite numbers greater than zero." + f"{archive_note} Correct the initialization file or archive metadata before rerunning." + ) + log_info(message, error=True) + if archive_error is not None: + raise ValueError(message) from archive_error + raise ValueError(message) + + for key in REQUIRED_TRANSIT_EPHEMERIS_FIELDS: + resolved[key] = _positive_finite_ephemeris_value(resolved[key]) + return resolved + + +# --------PLANETARY PARAMETERS UI------------------------------------------ +# Get the user's confirmation of values that will later be used in lightcurve fit +def get_planetary_parameters(candplanetbool, userpdict, pdict=None): + log_info("*******************************************") + log_info("Planetary Parameters for Lightcurve Fitting") + + # The order of planet_params list must match the pDict that is declared when scraping the NASA Exoplanet Archive + planet_params = ["Target Star RA in the form: HH:MM:SS (ignore the decimal values)", + "Target Star DEC in form: DD:MM:SS (ignore the decimal values and don't forget the '+' or '-' sign!)", + "Planet's Name", + "Host Star's Name", + "Orbital Period (days)", + "Orbital Period Uncertainty (days) \n(Keep in mind that 1.2e-34 is the same as 1.2 x 10^-34)", + "Published Mid-Transit Time (BJD_UTC)", + "Mid-Transit Time Uncertainty (BJD-UTC)", + "Ratio of Planet to Stellar Radius (Rp/Rs)", + "Ratio of Planet to Stellar Radius (Rp/Rs) Uncertainty", + "Ratio of Distance to Stellar Radius (a/Rs)", + "Ratio of Distance to Stellar Radius (a/Rs) Uncertainty", + "Orbital Inclination (deg)", + "Orbital Inclination (deg) Uncertainty", + "Argument of Periastron (deg)", + "Orbital Eccentricity (0 if null)", + "Star Effective Temperature (K)", + "Star Effective Temperature Positive Uncertainty (K)", + "Star Effective Temperature Negative Uncertainty (K)", + "Star Metallicity ([FE/H])", + "Star Metallicity Positive Uncertainty ([FE/H])", + "Star Metallicity Negative Uncertainty ([FE/H])", + "Star Surface Gravity (log(g))", + "Star Surface Gravity Positive Uncertainty (log(g))", + "Star Surface Gravity Negative Uncertainty (log(g))", + "Star Distance (pc)", + "Star Proper Motion RA (mas/yr)", + "Star Proper Motion DEC (mas/yr)"] + + # Conversion between hours to degrees if user entered ra and dec + if userpdict['ra'] is None: + userpdict['ra'] = user_input(f"\nEnter the {planet_params[0]}: ", type_=str) + if userpdict['dec'] is None: + userpdict['dec'] = user_input(f"\nEnter the {planet_params[1]}: ", type_=str) + if isinstance(userpdict['ra'], str) or isinstance(userpdict['dec'], str): + userpdict['ra'], userpdict['dec'] = radec_hours_to_degree(userpdict['ra'], userpdict['dec']) + + radeclist = ['ra', 'dec'] + if not candplanetbool: + for idx, item in enumerate(radeclist): + uncert = 20 / 3600 + if pdict[item] - uncert <= userpdict[item] <= pdict[item] + uncert: + continue + else: + log_info(f"\n\nWarning: {pdict['pName']} initialization file's {planet_params[idx]} does not match " + "the value scraped by EXOTIC from the NASA Exoplanet Archive.\n", warn=True) + log_info(f"\tNASA Exoplanet Archive value (degrees): {pdict[item]}", warn=True) + log_info(f"\tInitialization file value (degrees): {userpdict[item]}", warn=True) + log_info("\nWould you like to:" + "\n (1) use NASA Exoplanet Archive value, " + "\n (2) use initialization file value, or " + "\n (3) enter in a new value.", warn=True) + option = user_input("Which option do you choose? (1/2/3): ", type_=int, values=[1, 2, 3]) + + if option == 1: + userpdict[item] = pdict[item] + elif option == 2: + continue + else: + userpdict['ra'] = user_input(f"Enter the {planet_params[0]}: ", type_=str) + userpdict['dec'] = user_input(f"Enter the {planet_params[1]}: ", type_=str) + break + + if isinstance(userpdict['ra'], str) or isinstance(userpdict['dec'], str): + userpdict['ra'], userpdict['dec'] = radec_hours_to_degree(userpdict['ra'], userpdict['dec']) + + # Exoplanet confirmed in NASA Exoplanet Archive + if not candplanetbool: log_info(f"*** Here are the values scraped from the NASA Exoplanet Archive for {pdict['pName']} that were not " "set (or set to null) in your initialization file. ***") - for i, key in enumerate(userpdict): - if key in ('ra', 'dec'): - continue - if key in ('pName', 'sName'): - userpdict[key] = pdict[key] - # Initialization planetary parameters match NEA - if pdict[key] == userpdict[key]: - continue - # Initialization planetary parameters don't match NASA Exoplanet Archive - if userpdict[key] is not None: - log_info(f"\n\nWarning: {pdict['pName']} initialization file's {planet_params[i]} does not match " - "the value scraped by EXOTIC from the NASA Exoplanet Archive.\n", warn=True) - log_info(f"\tNASA Exoplanet Archive value: {pdict[key]}", warn=True) - log_info(f"\tInitialization file value: {userpdict[key]}", warn=True) - log_info("\nWould you like to: " - "\n (1) use NASA Exoplanet Archive value, " - "\n (2) use initialization file value, or " - "\n (3) enter in a new value.", warn=True) - option = user_input("Which option do you choose? (1/2/3): ", type_=int, values=[1, 2, 3]) - if option == 1: - userpdict[key] = pdict[key] - elif option == 2: - continue + for i, key in enumerate(userpdict): + if key in ('ra', 'dec'): + continue + if key in ('pName', 'sName'): + userpdict[key] = pdict[key] + # Initialization planetary parameters match NEA + if pdict[key] == userpdict[key]: + continue + # Initialization planetary parameters don't match NASA Exoplanet Archive + if userpdict[key] is not None: + log_info(f"\n\nWarning: {pdict['pName']} initialization file's {planet_params[i]} does not match " + "the value scraped by EXOTIC from the NASA Exoplanet Archive.\n", warn=True) + log_info(f"\tNASA Exoplanet Archive value: {pdict[key]}", warn=True) + log_info(f"\tInitialization file value: {userpdict[key]}", warn=True) + log_info("\nWould you like to: " + "\n (1) use NASA Exoplanet Archive value, " + "\n (2) use initialization file value, or " + "\n (3) enter in a new value.", warn=True) + option = user_input("Which option do you choose? (1/2/3): ", type_=int, values=[1, 2, 3]) + if option == 1: + userpdict[key] = pdict[key] + elif option == 2: + continue + else: + userpdict[key] = user_input(f"Enter the {planet_params[i]}: ", type_=type(userpdict[key])) + # Did not use initialization file or null + else: + log_info(f"\n {pdict['pName']} {planet_params[i]}: {pdict[key]}") + agreement = user_input("Do you agree? (y/n): ", type_=str, values=['y', 'n']) + if agreement == 'y': + userpdict[key] = pdict[key] + else: + userpdict[key] = user_input(f"Enter the {planet_params[i]}: ", type_=type(pdict[key])) + + # Exoplanet not confirmed in NASA Exoplanet Archive + else: + for i, key in enumerate(userpdict): + if key in ('ra', 'dec'): + continue + # Used initialization file and is not empty + if userpdict[key] is not None: + agreement = user_input(f"{planet_params[i]}: {userpdict[key]} \nDo you agree? (y/n): ", + type_=str, values=['y', 'n']) + if agreement == 'y': + continue + else: + userpdict[key] = user_input(f"Enter the {planet_params[i]}: ", type_=type(userpdict[key])) + # Did not use initialization file + else: + if key in ('pName', 'sName'): + userpdict[key] = user_input(f"\nEnter the {planet_params[i]}: ", type_=str) + else: + userpdict[key] = user_input(f"Enter the {planet_params[i]}: ", type_=float) + return userpdict + + +# Conversion of Right Ascension and Declination: hours -> degrees +def radec_hours_to_degree(ra, dec, non_interactive_run=False, archive_ra=None, archive_dec=None, + target_name=None): + def parse_coordinates(ra_input, dec_input): + ra_value = str(ra_input).strip() + dec_value = str(dec_input).strip() + + # Accept either sexagesimal RA strings (HH:MM:SS) or decimal RA degrees. + # A decimal-like value with no separators should be treated as degrees. + ra_unit = u.hourangle if any(sep in ra_value for sep in (':', ' ')) else u.deg + + # Declination can be provided as either sexagesimal or decimal degrees. + dec_unit = u.deg + if any(sep in dec_value for sep in (':', ' ')): + dec_value = dec_value.replace(':', ' ') + + if ra_unit is u.hourangle: + ra_value = ra_value.replace(':', ' ') + + coordinates = SkyCoord(ra=ra_value, dec=dec_value, unit=(ra_unit, dec_unit)) + ra_degrees = float(coordinates.ra.degree) + dec_degrees = float(coordinates.dec.degree) + if not np.isfinite(ra_degrees) or not np.isfinite(dec_degrees): + raise ValueError("RA and Dec must both be finite values") + return ra_degrees, dec_degrees + + while True: + try: + return parse_coordinates(ra, dec) + except (TypeError, ValueError) as input_error: + if non_interactive_run: + target_description = f" for target {target_name}" if target_name else "" + archive_coordinates_supplied = archive_ra is not None or archive_dec is not None + if archive_ra is not None and archive_dec is not None: + try: + fallback_ra, fallback_dec = parse_coordinates(archive_ra, archive_dec) + except (TypeError, ValueError) as archive_error: + raise ValueError( + f"Non-interactive run cancelled{target_description}: initialization-file RA={ra!r} " + f"and Dec={dec!r} are invalid ({input_error}), and the NASA Exoplanet Archive " + f"coordinates RA={archive_ra!r} and Dec={archive_dec!r} are also unusable " + f"({archive_error}). Provide valid target coordinates in the initialization file." + ) from archive_error + + log_info( + f"Warning: initialization-file RA={ra!r} and Dec={dec!r} are invalid" + f"{target_description} ({input_error}). Using NASA Exoplanet Archive coordinates " + f"RA={fallback_ra:.8f} deg, Dec={fallback_dec:.8f} deg instead.", + warn=True, + ) + return fallback_ra, fallback_dec + + archive_reason = ( + f"the NASA Exoplanet Archive returned incomplete coordinates " + f"(RA={archive_ra!r}, Dec={archive_dec!r})" + if archive_coordinates_supplied + else "NASA Exoplanet Archive coordinates are unavailable" + ) + raise ValueError( + f"Non-interactive run cancelled{target_description}: initialization-file RA={ra!r} " + f"and Dec={dec!r} are invalid ({input_error}), and {archive_reason}. Provide valid target " + "coordinates in the initialization file." + ) from input_error + + log_info("Error: The format entered for Right Ascension and/or Declination is not correct, " + "please try again.", error=True) + ra = input("Input the Right Ascension of target (HH:MM:SS): ") + dec = input("Input the Declination of target (DD:MM:SS): ") + + +def check_all_standard_filters(ld, observed_filter): + if ld.check_standard(observed_filter): + return True + elif observed_filter['filter']: + filter_name = observed_filter['filter'].lower().replace(' ', '') + filter_name = re.sub(ld_re_punct_p, '', filter_name) + filter_abbreviation = next((filter_abbr for filter_abbr in LimbDarkening.fwhm_names_nonspecific.keys() + if filter_name == filter_abbr.lower()), None) + filter_desc = next((filter_desc for filter_desc in LimbDarkening.fwhm_names_nonspecific.values() + if filter_name == re.sub(ld_re_punct_p, '', filter_desc.lower().replace(' ', ''))), + None) + + if filter_abbreviation: + observed_filter['filter'] = LimbDarkening.fwhm_names_nonspecific.get(filter_abbreviation) + observed_filter['name'] = filter_abbreviation + custom_range(ld, observed_filter) + return True + + if filter_desc: + observed_filter['filter'] = filter_desc + observed_filter['name'] = next((k for k, v in LimbDarkening.fwhm_names_nonspecific.items() if v == filter_desc)) + custom_range(ld, observed_filter) + return True + + return False + + +def custom_range(ld, observed_filter): + while True: + if ld.check_fwhm(observed_filter): + ld.set_filter(observed_filter['name'], observed_filter['filter'], + float(observed_filter['wl_min']), float(observed_filter['wl_max'])) + return + else: + observed_filter['wl_min'] = user_input(f"FWHM minimum wavelength (nm):", type_=str) + observed_filter['wl_max'] = user_input(f"FWHM maximum wavelength (nm):", type_=str) + + +def standard_filter(ld, observed_filter): + LimbDarkening.standard_list() + + while True: + if not observed_filter['filter']: + observed_filter['filter'] = user_input("\nPlease enter in the Filter Name or Abbreviation " + "(EX: Johnson V, V, STB, RJ): ", type_=str) + + if check_all_standard_filters(ld, observed_filter): + return + else: + log_info("\nError: The entered filter is not in the provided list of standard filters.", warn=True) + observed_filter['filter'] = None + + +def user_entered_ld(ld, observed_filter): + order = ['first', 'second', 'third', 'fourth'] + + input_list = [(f"\nEnter in your {order[i]} nonlinear term:", + f"\nEnter in your {order[i]} nonlinear term uncertainty:") for i in range(len(order))] + ld_ = [(user_input(input_[0], type_=float), user_input(input_[1], type_=float)) for input_ in input_list] + + custom_range(ld, observed_filter) + ld.set_ld(ld_[0], ld_[1], ld_[2], ld_[3]) + + +def nonlinear_ld(ld, info_dict, non_interactive_run=False): + user_entered = False + observed_filter = { + 'filter': info_dict['filter'], + 'name': None, + 'wl_min': info_dict['wl_min'], + 'wl_max': info_dict['wl_max'] + } + ld.check_fwhm(observed_filter) + + if not check_all_standard_filters(ld, observed_filter): + if observed_filter['wl_min'] and observed_filter['wl_max']: + custom_range(ld, observed_filter) + ld.set_filter('N/A', "Custom", float(observed_filter['wl_min']), float(observed_filter['wl_max'])) + else: + if non_interactive_run: + raise ValueError( + f"EXOTIC did not recognize the filter {info_dict.get('filter')!r}. " + "Non-interactive runs require a recognized standard filter or both wl_min and wl_max." + ) + + raw_opt = info_dict.get('ld_uncertainties') + opt = ( + None + if raw_opt is None or (isinstance(raw_opt, str) and not raw_opt.strip()) + else coerce_boolean_config_value(raw_opt) + ) + + if opt is None: + opt = user_input("\nWould you like EXOTIC to calculate your limb darkening parameters " + "with uncertainties? (y/n):", type_=str, values=['y', 'n']) + opt = coerce_boolean_config_value(opt) + + if opt: + opt = user_input("Please enter 1 to use a standard filter or 2 for a customized filter:", + type_=int, values=[1, 2]) + if opt == 1: + observed_filter['filter'] = None + standard_filter(ld, observed_filter) + elif opt == 2: + custom_range(ld, observed_filter) + ld.set_filter('N/A', "Custom", float(observed_filter['wl_min']), float(observed_filter['wl_max'])) + else: + user_entered_ld(ld, observed_filter) + user_entered = True + + if not user_entered: + ld.calculate_ld() + + info_dict['filter'] = ld.filter_name + info_dict['filter_desc'] = ld.filter_desc + info_dict['wl_min'] = ld.wl_min + info_dict['wl_max'] = ld.wl_max + + +def get_ld_values(planet_dict, info_dict, non_interactive_run=False): + ld_obj = LimbDarkening(planet_dict) + nonlinear_ld(ld_obj, info_dict, non_interactive_run=non_interactive_run) + + ld0 = ld_obj.ld0 + ld1 = ld_obj.ld1 + ld2 = ld_obj.ld2 + ld3 = ld_obj.ld3 + ld = [ld0[0], ld1[0], ld2[0], ld3[0]] + + return ld, ld0, ld1, ld2, ld3 + + +def corruption_check(files): + valid_files = [] + for file in files: + plateStatus.setCurrentFilename(file) + try: + with fits.open(name=file, memmap=False, cache=False, lazy_load_hdus=False, ignore_missing_end=True) as hdu1: + valid_files.append(file) + except OSError as e: + # Since google collab can have problems with initial load of big data sets from google + # drive, lets pause and retry this once when we fail: if the file was corrupted the first time, + # nothing will get better... + log_info(f"Warning: retrying verify\n\t-File: {file}\n\t-Reason: {e}", warn=True) + sleep(5) + try: + with fits.open(name=file, memmap=False, cache=False, lazy_load_hdus=False, ignore_missing_end=True) as hdu1: + valid_files.append(file) + except OSError as e: + log.debug(f"Warning: corrupted file found and removed from reduction\n\t-File: {file}\n\t-Reason: {e}") + plateStatus.fitsFormatError(e) + return valid_files + +def check_wcs(fits_file, save_directory, plate_opt, rt=False, use_nextastro_astrometry=False, + ra=None, dec=None, pixel_scale=None, ignore_header_wcs=False): + wcs_file = None + + if not ignore_header_wcs and search_wcs(fits_file).is_celestial: + if plate_opt == 'y' and not rt: + log_info("Your FITS files already have WCS (World Coordinate System) information in their headers. " + "EXOTIC will use the existing header WCS and skip external plate solving.") + else: + log_info("Your FITS files have WCS (World Coordinate System) information in their headers. " + "EXOTIC will proceed to use these. " + "NOTE: If you do not trust your WCS coordinates, " + "please restart EXOTIC after enabling plate solutions via astrometry.net.") + return fits_file + + if plate_opt == 'y' and not rt: + wcs_file = get_wcs(fits_file, save_directory, use_nextastro_astrometry=use_nextastro_astrometry, ra=ra, dec=dec, pixel_scale=pixel_scale) + if ignore_header_wcs: + if wcs_file: + log_info("Ignoring FITS header WCS for alignment and using the legacy image-to-image alignment path.") + else: + log_info("Ignoring FITS header WCS and using the legacy image-to-image alignment path.") + return wcs_file + + return wcs_file + + +def search_wcs(file): + header = get_first_image_header(file) + return search_wcs_from_header(header) + + +def search_wcs_from_header(header): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', category=FITSFixedWarning) + return WCS(header) + # return WCS(fits.open(file)[('SCI', 1)].header) + + +def get_first_image_header(file_name): + extension = 0 + header = fits.getheader(filename=file_name, ext=extension) + while header.get('NAXIS', 0) == 0: + extension += 1 + header = fits.getheader(filename=file_name, ext=extension) + return header + + +def collect_celestial_wcs_coverage(inputfiles): + has_celestial_wcs = [] + missing_wcs_files = [] + for file_name in inputfiles: + file_has_celestial_wcs = False + try: + image_header = get_first_image_header(file_name) + file_has_celestial_wcs = search_wcs_from_header(image_header).is_celestial + except Exception: + file_has_celestial_wcs = False + + has_celestial_wcs.append(file_has_celestial_wcs) + if not file_has_celestial_wcs: + missing_wcs_files.append(str(file_name)) + + return np.array(has_celestial_wcs, dtype=bool), missing_wcs_files + + +def evaluate_celestial_wcs_coverage(inputfiles): + has_celestial_wcs, missing_wcs_files = collect_celestial_wcs_coverage(inputfiles) + total_files = len(inputfiles) + all_have_celestial_wcs = total_files > 0 and bool(has_celestial_wcs.all()) + return all_have_celestial_wcs, missing_wcs_files + + +def log_file_preview(file_names, label): + if not file_names: + return + + preview = ", ".join([_display_filename(file_name) for file_name in file_names[:3]]) + remainder = len(file_names) - 3 + if remainder > 0: + preview = f"{preview}, ... (+{remainder} more)" + log.debug(f"{label}: {preview}") + + +def log_missing_celestial_wcs_preview(missing_wcs_files): + log_file_preview(missing_wcs_files, "Files without usable celestial WCS") + + +def format_file_preview_for_user(file_names, limit=6): + if not file_names: + return "" + + display_names = [_display_filename(file_name) for file_name in file_names[:limit]] + remainder = len(file_names) - len(display_names) + if remainder > 0: + display_names.append(f"... (+{remainder} more)") + return ", ".join(display_names) + + +def leading_rejected_reference_prefix(ordered_inputfiles, dropped_files): + if ordered_inputfiles is None: + return [], None + + ordered_inputfiles = [str(file_name) for file_name in ordered_inputfiles] + dropped_lookup = {str(file_name) for file_name in (dropped_files or [])} + leading_rejected = [] + next_candidate = None + + for file_name in ordered_inputfiles: + if file_name in dropped_lookup: + leading_rejected.append(file_name) + continue + next_candidate = file_name + break + + return leading_rejected, next_candidate + + +def reference_frame_rejection_fallback_info(reference_file, dropped_files, ordered_inputfiles=None, + rejection_label="Pointing precheck"): + if reference_file is None or not dropped_files: + return None + + reference_file = str(reference_file) + dropped_files = [str(file_name) for file_name in dropped_files] + if reference_file not in dropped_files: + return None + + other_dropped_files = [file_name for file_name in dropped_files if file_name != reference_file] + leading_rejected_files, next_reference_candidate = leading_rejected_reference_prefix( + ordered_inputfiles, + dropped_files, + ) + if not leading_rejected_files: + leading_rejected_files = [reference_file] + + log_info( + f"WARNING: {rejection_label} rejected the original reference image " + f"({_display_filename(reference_file)}). EXOTIC is automatically removing the leading rejected " + "frame(s) and continuing with a new reference image.", + warn=True, + ) + log_info( + "IMPORTANT: the supplied target and comparison-star pixel coordinates were tied to the rejected " + "reference image. EXOTIC will estimate the target pixel position from the target RA/Dec on the " + "new reference image and will replace the supplied comparison-star pixels with a new " + "image-detected comparison-star set using the same FITS-image criteria as nextastro_archive.", + warn=True, + ) + if other_dropped_files: + log_info( + f"{rejection_label} also rejected {len(other_dropped_files)} other frame(s): " + f"{format_file_preview_for_user(other_dropped_files)}", + warn=True, + ) + + leading_preview = format_file_preview_for_user(leading_rejected_files) + removal_instruction = ( + f"Automatically removed leading rejected frame(s) from this reduction: {leading_preview}." + ) + + if next_reference_candidate is not None: + removal_instruction += ( + f" Continuing from new reference image " + f"{_display_filename(next_reference_candidate)}." + ) + else: + removal_instruction += ( + " No non-rejected frame remains after that prefix, so this dataset does not have a usable " + "reference image." + ) + + log_info( + removal_instruction, + warn=True, + ) + return { + 'reference_file': reference_file, + 'leading_rejected_files': leading_rejected_files, + 'next_reference_candidate': next_reference_candidate, + 'other_dropped_files': other_dropped_files, + } + + +def abort_if_reference_frame_rejected(reference_file, dropped_files, ordered_inputfiles=None, + rejection_label="Pointing precheck"): + return reference_frame_rejection_fallback_info( + reference_file, + dropped_files, + ordered_inputfiles=ordered_inputfiles, + rejection_label=rejection_label, + ) is not None + + +def collect_wcs_frame_center_pointings(inputfiles): + positions = np.full((len(inputfiles), 2), np.nan, dtype=float) + usable_mask = np.zeros(len(inputfiles), dtype=bool) + usable_indices = [] + ra_values = [] + dec_values = [] + + for index, file_name in enumerate(inputfiles): + try: + image_header = get_first_image_header(file_name) + wcs = search_wcs_from_header(image_header) + if not wcs.is_celestial: + continue + + width = int(image_header.get("NAXIS1", 0)) + height = int(image_header.get("NAXIS2", 0)) + if width <= 0 or height <= 0: + continue + + center_x = (width - 1) / 2.0 + center_y = (height - 1) / 2.0 + ra_deg, dec_deg = wcs.pixel_to_world_values(center_x, center_y) + if not np.isfinite(ra_deg) or not np.isfinite(dec_deg): + continue + + usable_indices.append(index) + ra_values.append(float(ra_deg)) + dec_values.append(float(dec_deg)) + except Exception: + continue + + if not usable_indices: + return positions, usable_mask + + coords = SkyCoord(ra=np.asarray(ra_values) * u.deg, dec=np.asarray(dec_values) * u.deg, frame='icrs') + reference_coord = coords[0] + delta_lon, delta_lat = reference_coord.spherical_offsets_to(coords) + offsets = np.column_stack((delta_lon.to_value(u.arcsec), delta_lat.to_value(u.arcsec))) + + for index, offset in zip(usable_indices, offsets): + positions[index] = offset + usable_mask[index] = True + + return positions, usable_mask + + +def log_pointing_precheck_alignment_progress(i, total_files, file_name): + log_info( + f"Pointing precheck alignment progress: file {i + 1} of {total_files} : " + f"{_display_filename(file_name)}" + ) + + +def _pointing_precheck_return(positions, usable_mask, alignment_transforms, return_transforms): + if return_transforms: + return positions, usable_mask, alignment_transforms + return positions, usable_mask + + +def _filter_alignment_transform_cache(alignment_transforms, retained_files): + if not alignment_transforms: + return {} + + retained_keys = {str(file_name) for file_name in retained_files} + return { + file_key: tform + for file_key, tform in alignment_transforms.items() + if file_key in retained_keys + } + + +def collect_transform_frame_pointings(inputfiles, frame_loader=None, return_transforms=False, + multiprocess_transformations=None, + generalDark=None, generalBias=None, generalFlat=None, + demosaic_fmt=None, demosaic_out=None, demosaic_mult=None): + positions = np.full((len(inputfiles), 2), np.nan, dtype=float) + usable_mask = np.zeros(len(inputfiles), dtype=bool) + alignment_transforms = {} + + if len(inputfiles) == 0: + return _pointing_precheck_return(positions, usable_mask, alignment_transforms, return_transforms) + + if frame_loader is None: + frame_loader = lambda file_name: load_calibrated_reduction_image( + file_name, + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + ) + + total_files = len(inputfiles) + log_pointing_precheck_alignment_progress(0, total_files, inputfiles[0]) + try: + reference_image = frame_loader(inputfiles[0]) + except Exception as exc: + log_info( + f"Warning: pointing precheck alignment fallback could not load the reference frame " + f"{_display_filename(inputfiles[0])} ({exc}).", + warn=True, + ) + return _pointing_precheck_return(positions, usable_mask, alignment_transforms, return_transforms) + + if getattr(reference_image, "ndim", 0) != 2: + log_info("Warning: pointing precheck alignment fallback requires 2-D images; skipping.", warn=True) + return _pointing_precheck_return(positions, usable_mask, alignment_transforms, return_transforms) + + height, width = reference_image.shape + reference_anchor = np.array([[(width - 1) / 2.0, (height - 1) / 2.0]], dtype=float) + positions[0] = reference_anchor[0] + usable_mask[0] = True + alignment_transforms[str(inputfiles[0])] = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) + + if multiprocess_transformations is not None and multiprocess_transformations > 0 and len(inputfiles) > 1: + try: + positions, usable_mask, alignment_transforms = build_multiprocess_pointing_precheck_transforms( + inputfiles, + multiprocess_transformations, + reference_anchor, + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + demosaic_out=demosaic_out, + demosaic_mult=demosaic_mult, + ) + return _pointing_precheck_return(positions, usable_mask, alignment_transforms, return_transforms) + except Exception as exc: + log_info( + "Warning: pointing precheck multiprocessing failed; falling back to serial alignment " + f"({exc}).", + warn=True, + ) + + for index, file_name in enumerate(inputfiles[1:], start=1): + log_pointing_precheck_alignment_progress(index, total_files, file_name) + try: + image_data = frame_loader(file_name) + if getattr(image_data, "ndim", 0) != 2: + continue + + tform = downsampled_fallback_transformation( + image_data, + file_name, + report_failure=False, + reference_image=reference_image, + ) + mapped_anchor = np.asarray(tform(reference_anchor), dtype=float).reshape(-1, 2)[0] + if np.all(np.isfinite(mapped_anchor)): + positions[index] = mapped_anchor + usable_mask[index] = True + alignment_transforms[str(file_name)] = tform + except Exception: + continue + + return _pointing_precheck_return(positions, usable_mask, alignment_transforms, return_transforms) + + +def sigma_clip_pointing_positions(positions, sigma=3.0, max_iters=5): + positions = np.asarray(positions, dtype=float) + if positions.ndim != 2 or positions.shape[1] != 2: + raise ValueError("positions must be an Nx2 array") + + finite_mask = np.all(np.isfinite(positions), axis=1) + keep_mask = finite_mask.copy() + if np.count_nonzero(keep_mask) < POINTING_REJECTION_MIN_FRAMES: + return keep_mask + + sigma = float(sigma) + for _ in range(max_iters): + candidate_positions = positions[keep_mask] + if candidate_positions.shape[0] < POINTING_REJECTION_MIN_FRAMES: + break + + center = np.nanmedian(candidate_positions, axis=0) + deltas = candidate_positions - center + radial_offsets = np.hypot(deltas[:, 0], deltas[:, 1]) + + scatter_x = robust_scatter(deltas[:, 0]) + scatter_y = robust_scatter(deltas[:, 1]) + radial_scatter = robust_scatter(radial_offsets) + + if not np.isfinite(scatter_x) or scatter_x <= 0: + scatter_x = radial_scatter + if not np.isfinite(scatter_y) or scatter_y <= 0: + scatter_y = radial_scatter + + if (not np.isfinite(scatter_x) or scatter_x <= 0 + or not np.isfinite(scatter_y) or scatter_y <= 0): + break + + normalized_distance = np.sqrt((deltas[:, 0] / scatter_x) ** 2 + (deltas[:, 1] / scatter_y) ** 2) + current_keep = normalized_distance <= sigma + if np.all(current_keep): + break + + updated_keep = keep_mask.copy() + updated_keep[np.flatnonzero(keep_mask)] = current_keep + if np.array_equal(updated_keep, keep_mask): + break + keep_mask = updated_keep + + return keep_mask + + +def filter_pointing_outlier_frames(inputfiles, pointing_rejection_sigma=None, ignore_header_wcs=False, + allow_pixel_alignment_fallback=False, + frame_loader=None, return_alignment_transforms=False, + multiprocess_transformations=None, + generalDark=None, generalBias=None, generalFlat=None, + demosaic_fmt=None, demosaic_out=None, demosaic_mult=None): + inputfiles = np.array(inputfiles) + keep_mask = np.ones(len(inputfiles), dtype=bool) + alignment_transforms = {} + + def format_result(result_inputfiles, result_keep_mask, dropped_files): + if return_alignment_transforms: + return ( + result_inputfiles, + result_keep_mask, + dropped_files, + _filter_alignment_transform_cache(alignment_transforms, result_inputfiles), + ) + return result_inputfiles, result_keep_mask, dropped_files + + if len(inputfiles) == 0 or pointing_rejection_sigma is None: + return format_result(inputfiles, keep_mask, []) + + if len(inputfiles) < POINTING_REJECTION_MIN_FRAMES: + log_info( + f"Pointing precheck skipped: only {len(inputfiles)} frame(s); " + f"need at least {POINTING_REJECTION_MIN_FRAMES}.", + ) + return format_result(inputfiles, keep_mask, []) + + positions = None + usable_mask = None + mode_label = None + + pixel_alignment_enabled = bool(ignore_header_wcs or allow_pixel_alignment_fallback) + + if not ignore_header_wcs: + wcs_positions, wcs_usable_mask = collect_wcs_frame_center_pointings(inputfiles) + usable_wcs_count = int(np.count_nonzero(wcs_usable_mask)) + if usable_wcs_count == len(inputfiles): + positions = wcs_positions + usable_mask = wcs_usable_mask + mode_label = "WCS" + elif usable_wcs_count > 0 and pixel_alignment_enabled: + log_info( + f"Pointing precheck: usable WCS-derived pointing centers found for " + f"{usable_wcs_count}/{len(inputfiles)} frame(s); falling back to alignment-derived positions." + ) + elif pixel_alignment_enabled: + log_info("Pointing precheck: no usable WCS-derived pointing centers found; using alignment-derived positions.") + else: + positions = wcs_positions + usable_mask = wcs_usable_mask + mode_label = "WCS" + log_info( + f"Pointing precheck: usable WCS-derived pointing centers found for " + f"{usable_wcs_count}/{len(inputfiles)} frame(s). Pixel alignment fallback is disabled." + ) + + if positions is None: + positions, usable_mask, alignment_transforms = collect_transform_frame_pointings( + inputfiles, + frame_loader=frame_loader, + return_transforms=True, + multiprocess_transformations=multiprocess_transformations, + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + demosaic_out=demosaic_out, + demosaic_mult=demosaic_mult, + ) + mode_label = "alignment" + + usable_count = int(np.count_nonzero(usable_mask)) + if usable_count < POINTING_REJECTION_MIN_FRAMES: + log_info( + f"Pointing precheck skipped: only {usable_count} usable {mode_label}-derived pointing estimate(s); " + f"need at least {POINTING_REJECTION_MIN_FRAMES}.", + ) + return format_result(inputfiles, keep_mask, []) + + keep_mask[np.flatnonzero(usable_mask)] = sigma_clip_pointing_positions( + positions[usable_mask], + sigma=pointing_rejection_sigma, + max_iters=POINTING_REJECTION_MAX_ITERS, + ) + + dropped_files = inputfiles[~keep_mask].tolist() + if not dropped_files: + log_info( + f"Pointing precheck ({mode_label}): no frames exceeded the " + f"{float(pointing_rejection_sigma):g}-sigma pointing threshold." + ) + return format_result(inputfiles, keep_mask, []) + + retained_files = inputfiles[keep_mask] + log_info( + f"Pointing precheck ({mode_label}): {len(retained_files)}/{len(inputfiles)} frame(s) remain after " + f"dropping {len(dropped_files)} file(s) beyond {float(pointing_rejection_sigma):g} sigma from the " + "median pointing." + ) + log_file_preview(dropped_files, "Pointing precheck dropped files") + return format_result(retained_files, keep_mask, dropped_files) + + +def filter_sparse_missing_wcs_frames(inputfiles, ignore_header_wcs=False, max_missing_fraction=None, + allow_pixel_alignment_fallback=True): + inputfiles = np.array(inputfiles) + keep_mask = np.ones(len(inputfiles), dtype=bool) + if ignore_header_wcs or len(inputfiles) == 0: + return inputfiles, keep_mask, [] + + if max_missing_fraction is None: + max_missing_fraction = SPARSE_MISSING_WCS_DROP_THRESHOLD + + keep_mask, missing_wcs_files = collect_celestial_wcs_coverage(inputfiles) + missing_count = len(missing_wcs_files) + total_files = len(inputfiles) + if missing_count == 0: + return inputfiles, keep_mask, [] + + if not allow_pixel_alignment_fallback: + retained_files = inputfiles[keep_mask] + log_info( + f"WCS-authoritative precheck: {len(retained_files)}/{total_files} files have celestial WCS. " + f"Dropping all {missing_count} file(s) without celestial WCS because pixel alignment fallback " + "is disabled." + ) + log_missing_celestial_wcs_preview(missing_wcs_files) + return retained_files, keep_mask, missing_wcs_files + + missing_fraction = missing_count / total_files + if missing_count < total_files and missing_fraction < max_missing_fraction: + retained_files = inputfiles[keep_mask] + threshold_percent = max_missing_fraction * 100.0 + log_info( + f"WCS precheck: {len(retained_files)}/{total_files} files have celestial WCS. " + f"Dropping {missing_count} file(s) without celestial WCS because they are below the " + f"{threshold_percent:g}% threshold." + ) + log_missing_celestial_wcs_preview(missing_wcs_files) + return retained_files, keep_mask, missing_wcs_files + + threshold_percent = max_missing_fraction * 100.0 + log_info( + f"WCS precheck: {total_files - missing_count}/{total_files} files have celestial WCS. " + f"Retaining all {total_files} frame(s) and enabling pixel alignment fallback because the " + f"missing-WCS fraction is at or above the {threshold_percent:g}% threshold." + ) + log_missing_celestial_wcs_preview(missing_wcs_files) + return inputfiles, np.ones(total_files, dtype=bool), [] + + +def should_use_multiprocess_transform_precompute(inputfiles, requested_processes, ignore_header_wcs=False, + allow_pixel_alignment_fallback=True): + if requested_processes is None or requested_processes <= 0: + return False + + if ignore_header_wcs: + log_info("Header WCS ignore override enabled. Keeping multiprocessing transformation precompute.") + return True + + if not allow_pixel_alignment_fallback: + log_info("Pixel alignment fallback is disabled. Skipping multiprocessing transformation precompute.") + return False + + all_have_celestial_wcs, missing_wcs_files = evaluate_celestial_wcs_coverage(inputfiles) + if all_have_celestial_wcs: + log_info("All input FITS files have celestial WCS in their headers. " + "Skipping multiprocessing transformation precompute.") + return False + + total_files = len(inputfiles) + missing_count = len(missing_wcs_files) + log_info(f"WCS precheck: {total_files - missing_count}/{total_files} files have celestial WCS. " + "Keeping multiprocessing transformation precompute for fallback alignment.") + log_missing_celestial_wcs_preview(missing_wcs_files) + + return True + + +def get_wcs(file, directory="", use_nextastro_astrometry=False, ra=None, dec=None, pixel_scale=None): + astrometry_service = 'NextAstro astrometry server (https://astrometry.nextastro.org/)' if use_nextastro_astrometry else 'nova.astrometry.net' + log_info("\nGetting the plate solution for your imaging file to translate pixel coordinates on the sky. " + f"\nUsing astrometry service: {astrometry_service}." + "\nPlease wait....") + + if use_nextastro_astrometry: + print("Contacting NextAstro Astrometry Server") + nextastro_solver = NextAstroPlateSolution( + file=file, + directory=directory, + ra=ra, + dec=dec, + pixel_scale=pixel_scale, + suppress_fail_warning=True, + message_logger=log_info + ) + wcs_file = nextastro_solver.plate_solution() + if wcs_file: + return wcs_file + + nextastro_bad_gateway = nextastro_solver.last_http_status == 502 + if nextastro_bad_gateway: + log_info("NextAstro Server not responding. Will try nova.astrometry.net") + else: + log_info("NextAstro astrometry server did not return a solution; falling back to nova.astrometry.net.") + print("Communication with nova.astrometry.net") + nova_solver = PlateSolution(file=file, directory=directory, ra=ra, dec=dec, + pixel_scale=pixel_scale, suppress_fail_warning=True) + wcs_file = nova_solver.plate_solution() + if wcs_file: + return wcs_file + if nextastro_bad_gateway: + log_info("NextAstro Server not responding. Both astrometry methods trialed, pushing forward without astrometry solution") + return False + return PlateSolution.fail(nova_solver.last_error_type or 'plate solution lookup') + + animate_toggle(True) + nova_solver = PlateSolution(file=file, directory=directory, ra=ra, dec=dec, + pixel_scale=pixel_scale, suppress_fail_warning=True) + wcs_file = nova_solver.plate_solution() + if wcs_file: + animate_toggle() + return wcs_file + + log_info("nova.astrometry.net did not return a solution; falling back to NextAstro astrometry server.") + print("Contacting NextAstro Astrometry Server") + nextastro_solver = NextAstroPlateSolution( + file=file, + directory=directory, + ra=ra, + dec=dec, + pixel_scale=pixel_scale, + suppress_fail_warning=True, + message_logger=log_info + ) + wcs_file = nextastro_solver.plate_solution() + animate_toggle() + if wcs_file: + return wcs_file + if nextastro_solver.last_http_status == 502: + log_info("NextAstro Server not responding. Both astrometry methods trialed, pushing forward without astrometry solution") + return False + return PlateSolution.fail(nextastro_solver.last_error_type or 'plate solution lookup', + service_name=f'NextAstro ({nextastro_solver.api_url})') + + +# Getting the right ascension and declination for every pixel in imaging file if there is a plate solution +def _resolve_wcs_image_dimensions(header, image_shape=None): + width = header.get('NAXIS1', header.get('ZNAXIS1')) + height = header.get('NAXIS2', header.get('ZNAXIS2')) + if width is not None and height is not None: + return int(width), int(height) + + if image_shape is not None and len(image_shape) >= 2: + return int(image_shape[-1]), int(image_shape[-2]) + + wcs_header = WCS(header) + if wcs_header.pixel_shape is not None and len(wcs_header.pixel_shape) >= 2: + return int(wcs_header.pixel_shape[0]), int(wcs_header.pixel_shape[1]) + + if wcs_header.array_shape is not None and len(wcs_header.array_shape) >= 2: + return int(wcs_header.array_shape[1]), int(wcs_header.array_shape[0]) + + raise KeyError("Keyword 'NAXIS1' not found.") + + +def get_ra_dec(header, image_shape=None): + wcs_header = WCS(header) + width, height = _resolve_wcs_image_dimensions(header, image_shape=image_shape) + xaxis = np.arange(width) + yaxis = np.arange(height) + x, y = np.meshgrid(xaxis, yaxis) + # Image arrays and every pixel coordinate used by EXOTIC are zero-based. + # Passing origin=1 here displaced the sky-coordinate grid by one pixel in + # both axes and made precise catalog matches fail on coarse image scales. + return wcs_header.all_pix2world(x, y, 0) + + +def deg_to_pix(exp_ra, exp_dec, ra_list, dec_list): + dist = (ra_list - exp_ra) ** 2 + (dec_list - exp_dec) ** 2 + return np.unravel_index(dist.argmin(), dist.shape) + + +def project_target_pixel_wcs(exp_ra, exp_dec, ra_list, dec_list, wcs_header=None): + if wcs_header is not None: + try: + x_pixel, y_pixel = WCS(wcs_header).all_world2pix(exp_ra, exp_dec, 0) + x_pixel = float(np.asarray(x_pixel).reshape(-1)[0]) + y_pixel = float(np.asarray(y_pixel).reshape(-1)[0]) + if np.isfinite(x_pixel) and np.isfinite(y_pixel): + return x_pixel, y_pixel + except Exception as exc: + log.debug(f"Direct WCS pixel projection failed; falling back to grid search: {exc}") + + calculated_y_pixel, calculated_x_pixel = deg_to_pix(exp_ra, exp_dec, ra_list, dec_list) + return float(calculated_x_pixel), float(calculated_y_pixel) + + +def pixel_within_image(x_pixel, y_pixel, image_shape, margin=0.0): + height, width = image_shape[:2] + return ( + np.isfinite(x_pixel) + and np.isfinite(y_pixel) + and margin <= x_pixel < (width - margin) + and margin <= y_pixel < (height - margin) + ) + + +def any_projected_coord_out_of_frame(coords, image_shape): + for x_pixel, y_pixel in np.asarray(coords, dtype=float): + if not pixel_within_image(x_pixel, y_pixel, image_shape): + return True + return False + + +def project_ra_dec_to_wcs_pixel(ra, dec, wcs_header): + x_pixel, y_pixel = WCS(wcs_header).all_world2pix(ra, dec, 0) + x_pixel = float(np.asarray(x_pixel).reshape(-1)[0]) + y_pixel = float(np.asarray(y_pixel).reshape(-1)[0]) + return x_pixel, y_pixel + + +def project_comparison_radec_to_pixels(comp_stars_radec, wcs_header, image_shape): + """Project supplied celestial comparison coordinates onto a reference image.""" + projected = [] + for index, (ra_deg, dec_deg) in enumerate(comp_stars_radec or [], start=1): + try: + x_pixel, y_pixel = project_ra_dec_to_wcs_pixel(ra_deg, dec_deg, wcs_header) + except Exception as exc: + raise ValueError( + f"comparison star {index} RA/Dec could not be projected by the reference-image WCS" + ) from exc + if not pixel_within_image(x_pixel, y_pixel, image_shape): + raise ValueError( + f"comparison star {index} at RA={ra_deg:.8f}, Dec={dec_deg:.8f} " + "projects outside the reference image" + ) + projected.append([x_pixel, y_pixel]) + return projected + + +def _representative_obs_time(obs_times): + if obs_times is None: + return None + + try: + obs_times = np.asarray(obs_times, dtype=float).reshape(-1) + except (TypeError, ValueError): + return None + + finite_obs_times = obs_times[np.isfinite(obs_times)] + if finite_obs_times.size == 0: + return None + return float(np.nanmedian(finite_obs_times)) + + +def target_ra_dec_for_wcs_filter(info_dict, obs_times=None): + obs_time = _representative_obs_time(obs_times) + if obs_time is not None: + target_ra, target_dec = update_coordinates_with_proper_motion(info_dict, obs_time) + return float(target_ra), float(target_dec) + + return float(info_dict['ra']), float(info_dict['dec']) + + +def wcs_target_projection_status(image_header, target_ra, target_dec): + wcs = search_wcs_from_header(image_header) + if not wcs.is_celestial: + return None, None + + x_pixel, y_pixel = project_ra_dec_to_wcs_pixel(target_ra, target_dec, image_header) + width, height = _resolve_wcs_image_dimensions(image_header) + image_shape = (height, width) + return pixel_within_image(x_pixel, y_pixel, image_shape), (x_pixel, y_pixel) + + +def collect_wcs_target_coverage(inputfiles, info_dict, obs_times=None): + keep_mask = np.ones(len(inputfiles), dtype=bool) + dropped_files = [] + + try: + target_ra, target_dec = target_ra_dec_for_wcs_filter(info_dict, obs_times=obs_times) + except Exception as exc: + log_info( + "Warning: target WCS precheck could not determine target RA/Dec " + f"({exc}); skipping target-in-frame filtering.", + warn=True, + ) + return keep_mask, dropped_files + + for index, file_name in enumerate(inputfiles): + try: + image_header = get_first_image_header(file_name) + projection_inside, _ = wcs_target_projection_status(image_header, target_ra, target_dec) + except Exception: + projection_inside = False + + if projection_inside is False: + keep_mask[index] = False + dropped_files.append(str(file_name)) + + return keep_mask, dropped_files + + +def count_wcs_target_projection_hits(inputfiles, target_ra, target_dec): + celestial_count = 0 + inside_count = 0 + + for file_name in inputfiles: + try: + image_header = get_first_image_header(file_name) + projection_inside, _ = wcs_target_projection_status(image_header, target_ra, target_dec) + except Exception: + continue + + if projection_inside is None: + continue + + celestial_count += 1 + if projection_inside: + inside_count += 1 + + return inside_count, celestial_count + + +def maybe_reinterpret_decimal_ra_hours_from_wcs(inputfiles, info_dict, obs_times=None): + if len(inputfiles) == 0: + return False + + try: + original_ra = float(info_dict['ra']) + float(info_dict['dec']) + except (KeyError, TypeError, ValueError): + return False + + if not (0.0 <= original_ra <= 24.0): + return False + + try: + target_ra, target_dec = target_ra_dec_for_wcs_filter(info_dict, obs_times=obs_times) + except Exception: + return False + + primary_inside_count, celestial_count = count_wcs_target_projection_hits( + inputfiles, + target_ra, + target_dec, + ) + if celestial_count == 0 or primary_inside_count > 0: + return False + + alternate_info_dict = dict(info_dict) + alternate_info_dict['ra'] = original_ra * 15.0 + try: + alternate_ra, alternate_dec = target_ra_dec_for_wcs_filter( + alternate_info_dict, + obs_times=obs_times, + ) + except Exception: + return False + + alternate_inside_count, _ = count_wcs_target_projection_hits( + inputfiles, + alternate_ra, + alternate_dec, + ) + if alternate_inside_count <= primary_inside_count: + return False + + info_dict['ra'] = alternate_info_dict['ra'] + log_info( + "Target WCS precheck: interpreted decimal target RA as hours because the supplied RA projected " + f"into 0/{celestial_count} WCS frame(s), while RA*15 projected into " + f"{alternate_inside_count}/{celestial_count} frame(s). Using RA={info_dict['ra']:.7f} deg.", + warn=True, + ) + return True + + +def filter_wcs_target_out_of_frame_frames(inputfiles, info_dict, obs_times=None, ignore_header_wcs=False): + inputfiles = np.array(inputfiles) + keep_mask = np.ones(len(inputfiles), dtype=bool) + if ignore_header_wcs or len(inputfiles) == 0: + return inputfiles, keep_mask, [] + + keep_mask, dropped_files = collect_wcs_target_coverage( + inputfiles, + info_dict, + obs_times=obs_times, + ) + if not dropped_files: + return inputfiles, keep_mask, [] + + retained_files = inputfiles[keep_mask] + log_info( + f"Target WCS precheck: {len(retained_files)}/{len(inputfiles)} frame(s) remain after dropping " + f"{len(dropped_files)} file(s) where the target RA/Dec projects outside the image." + ) + log_file_preview(dropped_files, "Target WCS precheck dropped files") + return retained_files, keep_mask, dropped_files + + +def check_target_pixel_wcs(input_x_pixel, input_y_pixel, info_dict, ra_list, dec_list, image_data, obs_time, + non_interactive_run=False, wcs_header=None, + prefer_pixel_values_over_wcs_for_target=False): + """ + Verify the provided pixel coordinates match the target's right ascension and declination. + """ + updated_ra, updated_dec = update_coordinates_with_proper_motion(info_dict, obs_time) + + calculated_x_pixel, calculated_y_pixel = project_target_pixel_wcs( + updated_ra, updated_dec, ra_list, dec_list, wcs_header=wcs_header + ) + + if not pixel_within_image(calculated_x_pixel, calculated_y_pixel, image_data.shape): + log_info("Warning: WCS-derived target pixel coordinates fall outside the image; " + "keeping the input target coordinates.", warn=True) + return input_x_pixel, input_y_pixel + + centroid_margin = 7.5 + if not pixel_within_image(calculated_x_pixel, calculated_y_pixel, image_data.shape, margin=centroid_margin): + log_info("Warning: WCS-derived target pixel coordinates are too close to the image edge for " + "centroid fitting; keeping the input target coordinates.", warn=True) + return input_x_pixel, input_y_pixel + + wcs_psf_row = get_psf_fit_row(image_data, calculated_x_pixel, calculated_y_pixel) + centroid_x, centroid_y = wcs_psf_row[0], wcs_psf_row[1] + sigma_x, sigma_y = wcs_psf_row[3], wcs_psf_row[4] + wcs_psf_quality_score = psf_solution_quality_score( + wcs_psf_row, + seed_pos=[calculated_x_pixel, calculated_y_pixel], + ) + + input_psf_quality_score = np.inf + if pixel_within_image(input_x_pixel, input_y_pixel, image_data.shape, margin=centroid_margin): + input_psf_row = get_psf_fit_row(image_data, input_x_pixel, input_y_pixel) + input_psf_quality_score = psf_solution_quality_score( + input_psf_row, + seed_pos=[input_x_pixel, input_y_pixel], + ) + + return check_coordinates(input_x_pixel, input_y_pixel, centroid_x, centroid_y, sigma_x, sigma_y, + calculated_x_pixel, calculated_y_pixel, non_interactive_run=non_interactive_run, + prefer_pixel_values_over_wcs_for_target=prefer_pixel_values_over_wcs_for_target, + wcs_psf_quality_score=wcs_psf_quality_score, + input_psf_quality_score=input_psf_quality_score) + + +def get_psf_fit_row(image_data, x_pixel, y_pixel): + try: + return fit_centroid(image_data, [x_pixel, y_pixel], 0) + except Exception as exc: + log.debug(f"Centroid fit failed while validating WCS target coordinates: {exc}") + return _nan_psf_result() + + +def get_psf_parameters(image_data, x_pixel, y_pixel): + psf_data = get_psf_fit_row(image_data, x_pixel, y_pixel) + if not np.all(np.isfinite(psf_data[:5])): + return np.nan, np.nan, np.nan, np.nan + return psf_data[0], psf_data[1], psf_data[3], psf_data[4] + + +def check_coordinates(input_x_pixel, input_y_pixel, centroid_x, centroid_y, sigma_x, sigma_y, + calculated_x_pixel, calculated_y_pixel, non_interactive_run=False, + prefer_pixel_values_over_wcs_for_target=False, + wcs_psf_quality_score=None, + input_psf_quality_score=None): + while True: + try: + validate_pixel_coordinates(input_x_pixel, input_y_pixel, centroid_x, centroid_y, sigma_x, sigma_y) + return input_x_pixel, input_y_pixel + except ValueError: + if should_prefer_pixel_values_over_wcs_for_target(prefer_pixel_values_over_wcs_for_target): + log_info("Proceeding with provided target pixel coordinates because " + "prefer_pixel_values_over_wcs_for_target is enabled.", warn=True) + return input_x_pixel, input_y_pixel + if non_interactive_run: + if wcs_psf_quality_score is None: + wcs_psf_quality_score = psf_solution_quality_score( + [centroid_x, centroid_y, 1.0, sigma_x, sigma_y, 0.0, 0.0], + seed_pos=[calculated_x_pixel, calculated_y_pixel], + ) + try: + wcs_score = float(wcs_psf_quality_score) + except (TypeError, ValueError): + wcs_score = np.inf + try: + input_score = float(input_psf_quality_score) + except (TypeError, ValueError): + input_score = np.inf + + if ( + np.isfinite(input_score) + ): + log_info( + "Proceeding with provided target pixel coordinates because they produce a plausible " + "target PSF fit; the WCS-derived target fit points to a different source.", + warn=True, + ) + return input_x_pixel, input_y_pixel + + if np.isfinite(wcs_score) and np.isfinite(centroid_x) and np.isfinite(centroid_y): + log_info("Proceeding with WCS-derived centroided target coordinates due to " + "--non-interactive-run.", warn=True) + return centroid_x, centroid_y + log_info("Proceeding with WCS-derived target pixel coordinates due to " + "--non-interactive-run (centroid unavailable or implausible).", warn=True) + return calculated_x_pixel, calculated_y_pixel + new_x_pixel, new_y_pixel = prompt_user_for_coordinates(input_x_pixel, input_y_pixel, + calculated_x_pixel, calculated_y_pixel) + if new_x_pixel == input_x_pixel and new_y_pixel == input_y_pixel: + return input_x_pixel, input_y_pixel + else: + input_x_pixel, input_y_pixel = new_x_pixel, new_y_pixel + + +def validate_pixel_coordinates(input_x_pixel, input_y_pixel, centroid_x, centroid_y, sigma_x, sigma_y): + """ + Validating the provided pixel coordinates are within 5 PSF of the expected coordinates. + """ + x_min = centroid_x - (sigma_x * 5) + x_max = centroid_x + (sigma_x * 5) + y_min = centroid_y - (sigma_y * 5) + y_max = centroid_y + (sigma_y * 5) + + if not (x_min <= input_x_pixel <= x_max): + log_info("\nWarning: The X Pixel Coordinate entered does not match the target's Right Ascension.", warn=True) + raise ValueError + if not (y_min <= input_y_pixel <= y_max): + log_info("\nWarning: The Y Pixel Coordinate entered does not match the target's Declination.", warn=True) + raise ValueError + + +def prompt_user_for_coordinates(input_x_pixel, input_y_pixel, calculated_x_pixel, calculated_y_pixel): + log_info(f"Your input pixel coordinates: [{input_x_pixel}, {input_y_pixel}]") + log_info(f"EXOTIC's calculated pixel coordinates: [{calculated_x_pixel}, {calculated_y_pixel}]") + opt = user_input("Would you like to re-enter the pixel coordinates? (y/n): ", type_=str, values=['y', 'n']) + + if opt == 'y': + use_suggested = user_input( + f"Here are the suggested pixel coordinates:" + f" X Pixel: {calculated_x_pixel}" + f" Y Pixel: {calculated_y_pixel}" + "\nWould you like to use these? (y/n): ", + type_=str, values=['y', 'n'] + ) + + if use_suggested == 'y': + return calculated_x_pixel, calculated_y_pixel + else: + input_x_pixel = user_input("Please re-enter the target star's X Pixel Coordinate: ", type_=int) + input_y_pixel = user_input("Please re-enter the target star's Y Pixel Coordinate: ", type_=int) + + return input_x_pixel, input_y_pixel + + +# Checks if comparison star is variable via querying SIMBAD +def query_variable_star_apis(ra, dec): + # Convert comparison star coordinates from pixel to WCS + sample = SkyCoord(ra * u.deg, dec * u.deg, frame='fk5') + return vsx_variable(sample.ra.deg, sample.dec.deg) + # + # # Query SIMBAD and search identifier result table to determine if comparison star is variable in any form + # # This is a secondary check if GAIA query returns inconclusive results + # star_name = simbad_query(sample) + # if not star_name: + # log_info("Warning: Your comparison star cannot be resolved in the SIMBAD star database; " + # "EXOTIC cannot check if it is variable or not. " + # "\nEXOTIC will still include this star in the reduction. " + # "\nPlease proceed with caution as we cannot check for stellar variability.\n", warn=True) + # return False + # else: + # identifiers = Simbad.query_objectids(star_name) + # + # for currName in identifiers: + # if "V*" in currName[0]: + # return True + # return False + + +@retry(stop=stop_after_delay(30)) +def vsx_auid(ra, dec, radius=0.01, maglimit=14): + try: + url = f"https://www.aavso.org/vsx/index.php?view=api.list&ra={ra}&dec={dec}&radius={radius}&tomag={maglimit}&format=json" + result = requests.get(url, timeout=30) + result.raise_for_status() + vsx_objects = extract_vsx_objects(result.json()) + if not vsx_objects: + return False + return vsx_objects[0].get('AUID', False) or False + except Exception: + log.info("\nThe target star does not have an AUID.") + return False + + +def extract_vsx_objects(payload): + if not isinstance(payload, dict): + return [] + + vsx_objects = payload.get('VSXObjects', []) + if isinstance(vsx_objects, dict): + vsx_object = vsx_objects.get('VSXObject', []) + if isinstance(vsx_object, dict): + return [vsx_object] + if isinstance(vsx_object, list): + return vsx_object + return [] + + if isinstance(vsx_objects, list): + return vsx_objects + + return [] + + +def vsx_object_value(vsx_object, *keys): + if not isinstance(vsx_object, dict): + return None + normalized = {str(key).strip().lower(): value for key, value in vsx_object.items()} + for key in keys: + value = normalized.get(str(key).strip().lower()) + if value not in (None, ''): + return value + return None + + +def vsx_numeric_value(value): + if value is None: + return None + if isinstance(value, (int, float, np.number)): + parsed = float(value) + return parsed if np.isfinite(parsed) else None + match = re.search( + r"[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?", + str(value).replace(',', ''), + ) + if match is None: + return None + parsed = _finite_float(match.group(0)) + return float(parsed) if parsed is not None else None + + +def vsx_object_ra_dec(vsx_object): + ra_value = vsx_object_value(vsx_object, 'RA2000', 'RA', 'ra_deg') + dec_value = vsx_object_value(vsx_object, 'Declination2000', 'Dec2000', 'DEC', 'dec_deg') + if ra_value is None or dec_value is None: + return None + + ra_text = str(ra_value).strip() + dec_text = str(dec_value).strip() + sexagesimal_ra = ':' in ra_text or len(ra_text.split()) > 1 + try: + if sexagesimal_ra: + coordinate = SkyCoord(ra_text, dec_text, unit=(u.hourangle, u.deg), frame='fk5') + return float(coordinate.ra.deg), float(coordinate.dec.deg) + ra_deg = float(ra_text) + dec_deg = float(dec_text) + if np.isfinite(ra_deg) and np.isfinite(dec_deg): + return ra_deg, dec_deg + except (TypeError, ValueError): + pass + try: + coordinate = SkyCoord(ra_text, dec_text, unit=(u.hourangle, u.deg), frame='fk5') + return float(coordinate.ra.deg), float(coordinate.dec.deg) + except Exception: + return None + + +def vsx_object_period_days(vsx_object): + return vsx_numeric_value(vsx_object_value(vsx_object, 'Period', 'period_days')) + + +def vsx_object_amplitude_mag(vsx_object): + direct_amplitude = vsx_numeric_value( + vsx_object_value(vsx_object, 'Amplitude', 'amplitude_mag') + ) + if direct_amplitude is not None and direct_amplitude >= 0: + return float(direct_amplitude) + + maximum_magnitude = vsx_numeric_value( + vsx_object_value(vsx_object, 'MaxMag', 'MaximumMagnitude', 'max_mag') + ) + minimum_magnitude = vsx_numeric_value( + vsx_object_value(vsx_object, 'MinMag', 'MinimumMagnitude', 'min_mag') + ) + if maximum_magnitude is None or minimum_magnitude is None: + return None + return float(abs(minimum_magnitude - maximum_magnitude)) + + +def fortuitous_variable_category(period_days, amplitude_mag): + period = _finite_float(period_days) + amplitude = _finite_float(amplitude_mag) + if ( + period is not None + and period > 0 + and period <= FORTUITOUS_VARIABLE_OPTIMAL_MAX_PERIOD_DAYS + and amplitude is not None + and amplitude >= FORTUITOUS_VARIABLE_OPTIMAL_MIN_AMPLITUDE_MAG + ): + return 'optimal_variables' + return 'normal' + + +def nextastro_vsx_query_boxes(ra, dec, radius_degrees): + center_ra = float(ra) % 360.0 + center_dec = float(dec) + radius = max(0.0, float(radius_degrees)) + dec_min = max(-90.0, center_dec - radius) + dec_max = min(90.0, center_dec + radius) + cos_dec = abs(np.cos(np.deg2rad(center_dec))) + if cos_dec < 1.0e-12: + return [(0.0, 360.0, dec_min, dec_max)] + + ra_radius = min(180.0, radius / cos_dec) + if ra_radius >= 180.0: + return [(0.0, 360.0, dec_min, dec_max)] + ra_min = (center_ra - ra_radius) % 360.0 + ra_max = (center_ra + ra_radius) % 360.0 + if ra_min <= ra_max: + return [(ra_min, ra_max, dec_min, dec_max)] + return [ + (ra_min, 360.0, dec_min, dec_max), + (0.0, ra_max, dec_min, dec_max), + ] + + +def normalize_nextastro_vsx_row(row): + if not isinstance(row, dict): + return None + ra = _finite_float(vsx_object_value(row, 'ra_deg', 'RA2000', 'ra')) + dec = _finite_float(vsx_object_value(row, 'dec_deg', 'Declination2000', 'dec')) + if ra is None or dec is None: + return None + normalized_keys = {str(key).strip().lower() for key in row} + magnitude = vsx_object_value(row, 'max_mag', 'mag1', 'MaxMag') + magnitude_band = vsx_object_value(row, 'max_passband', 'mag1_band') + if magnitude not in (None, '') and magnitude_band not in (None, ''): + magnitude = f"{magnitude} {magnitude_band}" + minimum_magnitude = vsx_object_value(row, 'min_mag', 'MinMag') + minimum_band = vsx_object_value(row, 'min_passband') + if minimum_magnitude not in (None, '') and minimum_band not in (None, ''): + minimum_magnitude = f"{minimum_magnitude} {minimum_band}" + return { + **row, + 'Name': vsx_object_value(row, 'name', 'Name'), + 'OID': vsx_object_value(row, 'oid', 'OID'), + 'RA2000': float(ra), + 'Declination2000': float(dec), + 'VariabilityType': vsx_object_value(row, 'var_type', 'VariabilityType', 'Type'), + 'Period': vsx_object_value(row, 'period_days', 'Period'), + 'Amplitude': vsx_object_value(row, 'amplitude_mag', 'Amplitude'), + 'MaxMag': magnitude, + 'MinMag': minimum_magnitude, + 'Category': 'Variable', + '_vsx_source': 'nextastro_cache', + '_vsx_has_full_metadata': { + 'period_days', + 'amplitude_mag', + 'max_mag', + 'min_mag', + }.issubset(normalized_keys), + } + + +@retry(stop=stop_after_delay(30)) +def nextastro_vsx_field_query(ra, dec, radius_degrees): + rows = [] + for ra_min, ra_max, dec_min, dec_max in nextastro_vsx_query_boxes( + ra, + dec, + radius_degrees, + ): + payload = { + 'ra_min': float(ra_min), + 'ra_max': float(ra_max), + 'dec_min': float(dec_min), + 'dec_max': float(dec_max), + 'limit': NEXTASTRO_VSX_QUERY_LIMIT, + 'offset': 0, + 'include_table': True, + 'compact': False, + } + response = requests.post(NEXTASTRO_VSX_QUERY_URL, json=payload, timeout=30) + response.raise_for_status() + body = response.json() + if not isinstance(body, dict) or not isinstance(body.get('rows'), list): + raise RuntimeError("NextAstro VSX cache returned an unexpected response format.") + columns = body.get('columns') if isinstance(body.get('columns'), list) else [] + for raw_row in body['rows']: + if isinstance(raw_row, dict): + row = raw_row + elif isinstance(raw_row, (list, tuple)) and len(raw_row) == len(columns): + row = dict(zip(columns, raw_row)) + else: + continue + normalized = normalize_nextastro_vsx_row(row) + if normalized is not None: + rows.append(normalized) + + deduplicated = [] + seen = set() + for row in rows: + oid = vsx_object_value(row, 'OID', 'oid') + coordinates = vsx_object_ra_dec(row) + key = ( + str(oid).strip() if oid not in (None, '') else '', + round(coordinates[0], 7) if coordinates else None, + round(coordinates[1], 7) if coordinates else None, + ) + if key in seen: + continue + seen.add(key) + deduplicated.append(row) + return deduplicated + + +@retry(stop=stop_after_delay(30)) +def vsx_field_query(ra, dec, radius_degrees, maglimit=FORTUITOUS_VARIABLE_VSX_MAGNITUDE_LIMIT): + url = "https://www.aavso.org/vsx/index.php" + response = requests.get( + url, + params={ + 'view': 'api.list', + 'ra': float(ra), + 'dec': float(dec), + 'radius': float(radius_degrees), + 'tomag': float(maglimit), + 'format': 'json', + }, + timeout=30, + ) + response.raise_for_status() + return extract_vsx_objects(response.json()) + + +def angular_separation_arcsec(first_ra, first_dec, second_ra, second_dec): + first_ra_rad, first_dec_rad, second_ra_rad, second_dec_rad = np.deg2rad([ + first_ra, + first_dec, + second_ra, + second_dec, + ]) + delta_ra = second_ra_rad - first_ra_rad + delta_dec = second_dec_rad - first_dec_rad + haversine = ( + np.sin(delta_dec / 2.0) ** 2 + + np.cos(first_dec_rad) * np.cos(second_dec_rad) * np.sin(delta_ra / 2.0) ** 2 + ) + haversine = float(np.clip(haversine, 0.0, 1.0)) + return float(np.rad2deg(2.0 * np.arcsin(np.sqrt(haversine))) * 3600.0) + + +def enrich_nextastro_vsx_objects(nextastro_objects, aavso_objects, match_radius_arcsec=2.0): + aavso_by_oid = { + str(vsx_object_value(obj, 'OID', 'oid')).strip(): obj + for obj in aavso_objects + if vsx_object_value(obj, 'OID', 'oid') not in (None, '') + } + enriched = [] + for cached_object in nextastro_objects: + match = None + oid = vsx_object_value(cached_object, 'OID', 'oid') + if oid not in (None, ''): + match = aavso_by_oid.get(str(oid).strip()) + cached_coordinates = vsx_object_ra_dec(cached_object) + if match is None and cached_coordinates is not None: + nearest_distance = None + for aavso_object in aavso_objects: + aavso_coordinates = vsx_object_ra_dec(aavso_object) + if aavso_coordinates is None: + continue + distance = angular_separation_arcsec( + *cached_coordinates, + *aavso_coordinates, + ) + if distance <= float(match_radius_arcsec) and ( + nearest_distance is None or distance < nearest_distance + ): + match = aavso_object + nearest_distance = distance + if match is None: + enriched.append(cached_object) + else: + enriched.append({ + **cached_object, + **match, + '_vsx_source': 'nextastro_cache+aavso_metadata', + }) + return enriched + + +def vsx_field_query_with_preference( + ra, + dec, + radius_degrees, + maglimit=FORTUITOUS_VARIABLE_VSX_MAGNITUDE_LIMIT, + use_nextastro_vsx_cache_first=False): + if not use_nextastro_vsx_cache_first: + return vsx_field_query(ra, dec, radius_degrees, maglimit=maglimit) + + try: + cached_objects = nextastro_vsx_field_query(ra, dec, radius_degrees) + except Exception as exc: + log_info( + "Warning: NextAstro VSX cache-first lookup failed; falling back to AAVSO VSX " + f"({describe_retry_exception(exc)}).", + warn=True, + ) + return vsx_field_query(ra, dec, radius_degrees, maglimit=maglimit) + + if not cached_objects: + log_info( + "NextAstro VSX cache-first lookup returned no field objects; " + "checking AAVSO VSX as a completeness fallback." + ) + return vsx_field_query(ra, dec, radius_degrees, maglimit=maglimit) + + log_info( + f"NextAstro VSX cache-first lookup returned {len(cached_objects)} field object(s)." + ) + if all(bool(obj.get('_vsx_has_full_metadata')) for obj in cached_objects): + log_info( + "NextAstro VSX cache supplied the full period/amplitude metadata schema; " + "skipping AAVSO metadata enrichment." + ) + return cached_objects + try: + aavso_objects = vsx_field_query(ra, dec, radius_degrees, maglimit=maglimit) + except Exception as exc: + log_info( + "Warning: AAVSO metadata enrichment failed; continuing with NextAstro VSX " + f"cache coordinates and types ({describe_retry_exception(exc)}).", + warn=True, + ) + return cached_objects + return enrich_nextastro_vsx_objects(cached_objects, aavso_objects) + + +def estimated_magnitude_error_from_reference_count_rate( + reference_image, + x_pos, + y_pos, + exposure_seconds=1.0, + gain_e_per_adu=None): + if reference_image is None: + return None + data = np.asarray(reference_image, dtype=float) + if data.ndim != 2: + return None + x_pos = _finite_float(x_pos) + y_pos = _finite_float(y_pos) + if x_pos is None or y_pos is None: + return None + + aperture_radius = float(REFERENCE_FALLBACK_DETECTION_APERTURE_RADIUS_PIXELS) + aperture = CircularAperture(positions=[(x_pos, y_pos)], r=aperture_radius) + aperture_mask = aperture.to_mask(method='exact')[0] + aperture_cutout = aperture_mask.cutout(data, fill_value=np.nan) + if aperture_cutout is None: + return None + aperture_cutout = np.asarray(aperture_cutout, dtype=float) + aperture_weights = np.asarray(aperture_mask.data, dtype=float) + aperture_valid = ( + np.isfinite(aperture_cutout) + & np.isfinite(aperture_weights) + & (aperture_weights > 0) + ) + if not np.any(aperture_valid): + return None + + annulus_geometry = resolve_sky_annulus_geometry( + aperture_radius, + 3.0 * aperture_radius, + ) + sky_background, sky_sigma, sky_pixels = skybg_phot( + data, + -1, + x_pos, + y_pos, + r=annulus_geometry['inner_radius'], + dr=annulus_geometry['annulus_width'], + ) + if ( + not np.isfinite(sky_background) + or not np.isfinite(sky_sigma) + or not np.isfinite(sky_pixels) + or sky_pixels <= 0 + ): + return None + + aperture_pixels = float(np.sum(aperture_weights[aperture_valid])) + aperture_sum = float(np.sum( + aperture_weights[aperture_valid] * aperture_cutout[aperture_valid] + )) + flux = aperture_sum - (float(sky_background) * aperture_pixels) + if not np.isfinite(flux) or flux <= 0: + return None + exposure = _finite_float(exposure_seconds, 1.0) + if exposure is None or exposure <= 0: + exposure = 1.0 + noise_budget = compute_photometry_noise_budget( + flux, + sky_sigma, + aperture_pixels, + sky_pixels, + exposure_s=exposure, + airmass=1.0, + noise_config={'gain_e_per_adu': gain_e_per_adu}, + ) + flux_error = _finite_float(noise_budget.get('total')) + if flux_error is None or not np.isfinite(flux_error) or flux_error < 0: + return None + magnitude_error = (2.5 / np.log(10.0)) * flux_error / flux + if not np.isfinite(magnitude_error): + return None + return { + 'aperture_flux_adu': float(flux), + 'count_rate_adu_per_second': float(flux / exposure), + 'estimated_magnitude_error': float(magnitude_error), + 'reference_sky_background_adu_per_pixel': float(sky_background), + 'reference_sky_sigma_adu': float(sky_sigma), + 'reference_aperture_pixels': float(aperture_pixels), + 'reference_sky_pixels': float(sky_pixels), + 'reference_flux_error_adu': float(flux_error), + 'reference_noise_components_adu': { + key: float(value) + for key, value in noise_budget.items() + if np.isfinite(value) + }, + } + + +def discover_fortuitous_vsx_variables( + wcs_file, + image_shape, + img_scale, + reference_image, + obs_filter, + target_pixel=None, + field_catalog=None, + exposure_seconds=1.0, + gain_e_per_adu=None, + saturation_threshold=None, + maximum_magnitude_error=FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR, + use_nextastro_vsx_cache_first=USE_NEXTASTRO_VSX_CACHE_FIRST_DEFAULT): + if not wcs_file or reference_image is None or img_scale is None: + return [] + try: + image_height, image_width = image_shape[:2] + image_scale = float(img_scale) + wcs_header = search_wcs(wcs_file) + center_ra, center_dec = wcs_header.pixel_to_world_values( + float(image_width) / 2.0, + float(image_height) / 2.0, + ) + radius_degrees = ( + 0.5 * image_scale * float(np.hypot(image_width, image_height)) + + NEXTASTRO_PHOTOMETRY_FIELD_PADDING_ARCSEC + ) / 3600.0 + except Exception as exc: + log_info( + f"Warning: could not define the VSX field footprint for fortuitous photometry ({exc}).", + warn=True, + ) + return [] + + try: + vsx_objects = vsx_field_query_with_preference( + center_ra, + center_dec, + radius_degrees, + use_nextastro_vsx_cache_first=use_nextastro_vsx_cache_first, + ) + except Exception as exc: + log_info( + "Warning: full-field VSX lookup for fortuitous variables failed " + f"({describe_retry_exception(exc)}).", + warn=True, + ) + return [] + + target_values = None + try: + candidate_target = np.asarray(target_pixel, dtype=float).reshape(-1) + if candidate_target.size >= 2 and np.all(np.isfinite(candidate_target[:2])): + target_values = candidate_target[:2] + except (TypeError, ValueError): + target_values = None + + variables = [] + for vsx_object in vsx_objects: + vsx_category = str(vsx_object_value(vsx_object, 'Category') or '').strip().lower() + if vsx_category and vsx_category != 'variable': + continue + coordinates = vsx_object_ra_dec(vsx_object) + if coordinates is None: + continue + ra_deg, dec_deg = coordinates + try: + x_pos, y_pos = wcs_header.world_to_pixel_values(ra_deg, dec_deg) + x_pos = float(np.asarray(x_pos).reshape(-1)[0]) + y_pos = float(np.asarray(y_pos).reshape(-1)[0]) + except Exception: + continue + if not pixel_within_image( + x_pos, + y_pos, + image_shape, + margin=REFERENCE_FALLBACK_DETECTION_APERTURE_RADIUS_PIXELS + 2, + ): + continue + if target_values is not None: + target_distance = float(np.hypot(x_pos - target_values[0], y_pos - target_values[1])) + if target_distance <= REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS: + continue + if ( + _finite_float(saturation_threshold) is not None + and aperture_contains_overexposed_pixel( + reference_image, + x_pos, + y_pos, + REFERENCE_FALLBACK_DETECTION_APERTURE_RADIUS_PIXELS, + float(saturation_threshold), + ) + ): + continue + + count_rate_estimate = estimated_magnitude_error_from_reference_count_rate( + reference_image, + x_pos, + y_pos, + exposure_seconds=exposure_seconds, + gain_e_per_adu=gain_e_per_adu, + ) + if count_rate_estimate is None: + continue + if count_rate_estimate['estimated_magnitude_error'] >= float(maximum_magnitude_error): + continue + + if any( + np.hypot(existing['x'] - x_pos, existing['y'] - y_pos) + <= REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS + for existing in variables + ): + continue + + catalog_match = None + if field_catalog is not None: + catalog_match = nextastro_photometry_catalog_match( + field_catalog, + ra_deg, + dec_deg, + obs_filter, + ) + period_days = vsx_object_period_days(vsx_object) + amplitude_mag = vsx_object_amplitude_mag(vsx_object) + name = str( + vsx_object_value(vsx_object, 'Name', 'name', 'Identifier') + or f"VSX J{ra_deg:.6f}{dec_deg:+.6f}" + ).strip() + variables.append({ + 'name': name, + 'auid': vsx_object_value(vsx_object, 'AUID', 'auid'), + 'variable_type': vsx_object_value(vsx_object, 'Type', 'VariabilityType', 'VarType'), + 'period_days': period_days, + 'amplitude_mag': amplitude_mag, + 'category': fortuitous_variable_category(period_days, amplitude_mag), + 'ra': float(ra_deg), + 'dec': float(dec_deg), + 'x': float(x_pos), + 'y': float(y_pos), + 'pos': [float(x_pos), float(y_pos)], + 'catalog_match': catalog_match, + **count_rate_estimate, + }) + + variables.sort(key=lambda variable: ( + 0 if variable['category'] == 'optimal_variables' else 1, + variable['estimated_magnitude_error'], + variable['name'], + )) + log_info( + "Fortuitous-variable VSX field search retained " + f"{len(variables)} star(s) with reference-frame estimated errors below " + f"{float(maximum_magnitude_error):.3f} mag." + ) + return variables + + +def describe_retry_exception(err): + if isinstance(err, RetryError): + last_attempt = getattr(err, 'last_attempt', None) + attempt_number = getattr(last_attempt, 'attempt_number', None) + try: + root_cause = last_attempt.exception() if last_attempt is not None else None + except Exception: + root_cause = None + + if root_cause is not None: + attempt_text = f" after {attempt_number} attempts" if attempt_number else "" + return (f"{err.__class__.__name__}{attempt_text} " + f"({root_cause.__class__.__name__}: {root_cause})") + + return f"{err.__class__.__name__}: {err}" + + +def extract_http_status_code_from_error(err): + if err is None: + return None + + response = getattr(err, 'response', None) + status_code = getattr(response, 'status_code', None) + if status_code is not None: + try: + return int(status_code) + except (TypeError, ValueError): + return None + + status_match = re.search(r"\bHTTP\s+(\d{3})\b", str(err), flags=re.IGNORECASE) + if status_match is None: + return None + + try: + return int(status_match.group(1)) + except (TypeError, ValueError): + return None + + +def should_retry_nextastro_variability_error(err): + if isinstance(err, requests.exceptions.RequestException): + status_code = extract_http_status_code_from_error(err) + return status_code is None or status_code in NEXTASTRO_VARIABILITY_RETRYABLE_HTTP_STATUS_CODES + + if isinstance(err, RuntimeError): + status_code = extract_http_status_code_from_error(err) + return status_code in NEXTASTRO_VARIABILITY_RETRYABLE_HTTP_STATUS_CODES + + return False + + +def submit_nextastro_variability_request(api_url, payload, content_encoding=None): + request_body, headers, content_encoding, raw_size, compressed_size = build_compressed_json_request( + payload, + content_encoding=content_encoding, + ) + log_info( + "NextAstro variability request compression: " + f"{content_encoding} ({compressed_size} bytes sent; {raw_size} bytes raw)" + ) + return requests.post(api_url, data=request_body, headers=headers, timeout=30), content_encoding + + +@retry(stop=stop_after_delay(30)) +def vsx_variable(ra, dec, radius=0.01, maglimit=14): + default_vsx_error = None + try: + url = f"https://www.aavso.org/vsx/index.php?view=api.list&ra={ra}&dec={dec}&radius={radius}&tomag={maglimit}&format=json" + result = requests.get(url, timeout=30) + result.raise_for_status() + vsx_objects = extract_vsx_objects(result.json()) + if not vsx_objects: + return False + + first_vsx_object = vsx_objects[0] + var = first_vsx_object.get('Category', '') + + if isinstance(var, str) and var.lower() == "variable": + vname = first_vsx_object.get('Name') + vdec = first_vsx_object.get('Declination2000') + vra = first_vsx_object.get('RA2000') + log_info(f"\nVSX variable check found {vname} at RA {vra}, DEC {vdec}\n" + f"and will be removed from reduction.", warn=True) + return True + return False + except Exception as err: + default_vsx_error = err + + try: + log_info(f"\nDefault VSX request failed ({default_vsx_error}); falling back to NextAstro VSX server.", warn=True) + fallback_result = nextastro_variability_test([(ra, dec)]) + is_variable = bool(fallback_result[0]) + if is_variable: + log_info("\nNextAstro VSX fallback flagged this star as variable and it will be removed from reduction.", warn=True) + return is_variable + except Exception: + return False + +def build_comp_ra_dec(ra_wcs, dec_wcs, comp_stars): + comp_ra_dec = [] + for _, comp_star in enumerate(comp_stars[:]): + comp_ra_dec.append([ra_wcs[int(comp_star[1])][int(comp_star[0])], + dec_wcs[int(comp_star[1])][int(comp_star[0])]]) + return comp_ra_dec + + +@retry( + stop=stop_after_attempt(NEXTASTRO_VARIABILITY_MAX_RETRY_ATTEMPTS), + wait=wait_fixed(NEXTASTRO_VARIABILITY_RETRY_WAIT_SECONDS), + retry=retry_if_exception(should_retry_nextastro_variability_error), +) +def nextastro_variability_test(comp_ra_dec): + api_url = 'https://photometry.nextastro.org/variability_test' + + payload = [{'ra': float(ra), 'dec': float(dec)} for ra, dec in comp_ra_dec] + log_info(f"NextAstro variability request JSON: {json.dumps(payload)}") + result, content_encoding = submit_nextastro_variability_request(api_url, payload) + if result.status_code == 415 and content_encoding == 'zstd': + log_info( + "NextAstro variability server rejected zstd-compressed request (HTTP 415); " + "retrying this request once with gzip.", + warn=True, + ) + result, content_encoding = submit_nextastro_variability_request( + api_url, + payload, + content_encoding='gzip', + ) + if result.status_code != 200: + raise RuntimeError(f"NextAstro variability server returned HTTP {result.status_code}.") + + body = result.json() + log_info(f"NextAstro variability response JSON: {json.dumps(body)}") + if not isinstance(body, list) or len(body) != len(payload): + raise RuntimeError("NextAstro variability server returned an unexpected response format.") + + variability_flags = [] + for index, star in enumerate(body): + is_in_vsx = int(star.get('is_in_vsx', 0)) + if is_in_vsx not in [0, 1]: + raise RuntimeError(f"Unexpected is_in_vsx value ({is_in_vsx}) for star index {index}.") + variability_flags.append(bool(is_in_vsx)) + + return variability_flags + + +def _finite_float(value, default=None): + try: + parsed = float(value) + except (TypeError, ValueError): + return default + return parsed if np.isfinite(parsed) else default + + +def usable_catalog_reference_magnitude(magnitude, magnitude_error, + max_error=CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX): + parsed_magnitude = _finite_float(magnitude) + parsed_error = normalized_magnitude_error(magnitude_error) + error_limit = _finite_float(max_error) + if error_limit is None: + error_limit = CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX + if ( + not is_usable_apparent_magnitude(parsed_magnitude) + or parsed_error is None + or parsed_error > error_limit + ): + return None + return parsed_magnitude, parsed_error + + +def normalize_nextastro_filter_key(obs_filter): + return re.sub(r"[^a-z0-9]", "", str(obs_filter or "").lower()) + + +def observed_filter_uses_clear_v_calibration(obs_filter): + raw_filter = str(obs_filter or '').strip() + clear_v_filter_keys = { + 'cv', + 'clearv', + 'clearunfilteredreducedtovsequence', + 'mobscv', + 'c', + 'clear', + 'lum', + 'luminance', + 'w', + 'pl', + 'photographicg', + 'gaiag', + 'pg', + 'g1', + 'g2', + } + # Exact uppercase G is EXOTIC's short alias for Photographic G. Lowercase + # g remains the distinct Sloan-like catalogue band. + return ( + raw_filter == 'G' + or normalize_nextastro_filter_key(raw_filter) in clear_v_filter_keys + ) + + +def nextastro_catalog_match_radius_arcsec(img_scale=None): + """Allow at least one image pixel when matching pixel-derived sky positions.""" + pixel_scale_arcsec = _finite_float(img_scale) + if pixel_scale_arcsec is None or pixel_scale_arcsec <= 0: + return NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC + return max(NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC, float(pixel_scale_arcsec)) + + +def reported_stellar_variability_band(observed_filter, fallback_band=None): + """Keep the catalogue anchor band separate from the measured passband.""" + if observed_filter_uses_clear_v_calibration(observed_filter): + return 'ClearV' + reported_band = fallback_band or observed_filter or 'V' + if str(reported_band).strip().lower() == 'r': + return 'rp' + return reported_band + + +def nextastro_photometry_band_candidates(obs_filter): + """Return the one explicitly configured catalogue calibration band. + + Absolute calibration never falls through to another band when that + configured magnitude or uncertainty is unavailable. + """ + filter_key = normalize_nextastro_filter_key(obs_filter) + if observed_filter_uses_clear_v_calibration(obs_filter): + return [('Vmag', 'err_Vmag', 'V')] + direct_map = { + 'u': [('umag', 'err_umag', 'u')], + 'johnsonu': [('umag', 'err_umag', 'u')], + 'su': [('umag', 'err_umag', 'u')], + 'up': [('umag', 'err_umag', 'u')], + 'b': [('Bmag', 'err_Bmag', 'B')], + 'johnsonb': [('Bmag', 'err_Bmag', 'B')], + 'photographicb': [('Bmag', 'err_Bmag', 'B')], + 'bb': [('Bmag', 'err_Bmag', 'B')], + 'pb': [('Bmag', 'err_Bmag', 'B')], + 'v': [('Vmag', 'err_Vmag', 'V')], + 'johnsonv': [('Vmag', 'err_Vmag', 'V')], + 'bv': [('Vmag', 'err_Vmag', 'V')], + 'cv': [('Vmag', 'err_Vmag', 'V')], + 'clearv': [('Vmag', 'err_Vmag', 'V')], + 'clearunfilteredreducedtovsequence': [('Vmag', 'err_Vmag', 'V')], + 'mobscv': [('Vmag', 'err_Vmag', 'V')], + 'c': [('Vmag', 'err_Vmag', 'V')], + 'clear': [('Vmag', 'err_Vmag', 'V')], + 'lum': [('Vmag', 'err_Vmag', 'V')], + 'luminance': [('Vmag', 'err_Vmag', 'V')], + 'sg': [('g', 'dg', 'g')], + 'sloang': [('g', 'dg', 'g')], + 'sdssg': [('g', 'dg', 'g')], + 'gp': [('g', 'dg', 'g')], + 'g': [('g', 'dg', 'g')], + 'sr': [('r', 'dr', 'r')], + 'sloanr': [('r', 'dr', 'r')], + 'sdssr': [('r', 'dr', 'r')], + 'johnsonr': [('r', 'dr', 'r')], + 'cousinsr': [('r', 'dr', 'r')], + 'clearunfilteredreducedtorsequence': [('r', 'dr', 'r')], + 'photographicr': [('r', 'dr', 'r')], + 'rp': [('r', 'dr', 'r')], + 'r': [('r', 'dr', 'r')], + 'rc': [('r', 'dr', 'r')], + 'rj': [('r', 'dr', 'r')], + 'pr': [('r', 'dr', 'r')], + 'tr': [('r', 'dr', 'r')], + 'cr': [('r', 'dr', 'r')], + 'si': [('i', 'di', 'i')], + 'sloani': [('i', 'di', 'i')], + 'sdssi': [('i', 'di', 'i')], + 'johnsoni': [('i', 'di', 'i')], + 'cousinsi': [('i', 'di', 'i')], + 'ip': [('i', 'di', 'i')], + 'i': [('i', 'di', 'i')], + 'ic': [('i', 'di', 'i')], + 'ij': [('i', 'di', 'i')], + 'sz': [('z', 'dz', 'z')], + 'sloanz': [('z', 'dz', 'z')], + 'sdssz': [('z', 'dz', 'z')], + 'panstarrszshort': [('z', 'dz', 'z')], + 'zp': [('z', 'dz', 'z')], + 'z': [('z', 'dz', 'z')], + 'zs': [('z', 'dz', 'z')], + } + + return list(direct_map.get(filter_key, [])) + + +def nextastro_photometry_lookup_columns(obs_filter): + band_candidates = nextastro_photometry_band_candidates(obs_filter) + if not band_candidates: + return None + magnitude_column, error_column, _ = band_candidates[0] + return { + 'columns': [ + *NEXTASTRO_PHOTOMETRY_IDENTITY_COLUMNS, + magnitude_column, + error_column, + ], + 'required_columns': [magnitude_column, error_column], + } + + +def aavso_vsp_band_for_filter(obs_filter): + if observed_filter_uses_clear_v_calibration(obs_filter): + return 'V' + filter_key = normalize_nextastro_filter_key(obs_filter) + direct_map = { + 'u': 'U', + 'johnsonu': 'U', + 'bu': 'U', + 'b': 'B', + 'johnsonb': 'B', + 'photographicb': 'B', + 'bb': 'B', + 'pb': 'B', + 'v': 'V', + 'johnsonv': 'V', + 'bv': 'V', + 'cv': 'V', + 'clearv': 'V', + 'clearunfilteredreducedtovsequence': 'V', + 'mobscv': 'V', + 'c': 'V', + 'clear': 'V', + 'lum': 'V', + 'luminance': 'V', + 'r': 'Rc', + 'rc': 'Rc', + 'cousinsr': 'Rc', + 'clearunfilteredreducedtorsequence': 'Rc', + 'cr': 'Rc', + 'i': 'Ic', + 'ic': 'Ic', + 'cousinsi': 'Ic', + 'su': 'SU', + 'sloanu': 'SU', + 'up': 'SU', + 'sg': 'SG', + 'sloang': 'SG', + 'sdssg': 'SG', + 'gp': 'SG', + 'sr': 'SR', + 'sloanr': 'SR', + 'sdssr': 'SR', + 'rp': 'SR', + 'si': 'SI', + 'sloani': 'SI', + 'sdssi': 'SI', + 'ip': 'SI', + 'sz': 'SZ', + 'sloanz': 'SZ', + 'sdssz': 'SZ', + 'zp': 'SZ', + } + return direct_map.get(filter_key, obs_filter) + + +def nextastro_catalog_rows(catalog_response): + if not isinstance(catalog_response, dict): + return [] + rows = catalog_response.get('rows', []) + if not isinstance(rows, list): + return [] + columns = catalog_response.get('columns', []) + if catalog_response.get('row_format') == 'arrays': + return [ + {column: row[index] if index < len(row) else None for index, column in enumerate(columns)} + for row in rows + if isinstance(row, list) + ] + return [row for row in rows if isinstance(row, dict)] + + +def row_nextastro_magnitude(row, band_candidates, max_error=CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX): + base_error_limit = _finite_float(max_error) + if base_error_limit is None: + base_error_limit = CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX + for priority, (mag_column, error_column, band_label) in enumerate(band_candidates): + allow_bv_error_fallback = ( + str(band_label).strip().upper() in {'B', 'V'} + and base_error_limit == CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX + ) + effective_error_limit = ( + CATALOG_BV_REFERENCE_MAGNITUDE_ERROR_FALLBACK_MAX + if allow_bv_error_fallback + else base_error_limit + ) + usable_magnitude = usable_catalog_reference_magnitude( + row.get(mag_column), + row.get(error_column), + max_error=effective_error_limit, + ) + if usable_magnitude is None: + continue + magnitude, magnitude_error = usable_magnitude + return { + 'priority': priority, + 'magnitude_error_tier': int(magnitude_error > base_error_limit), + 'mag': magnitude, + 'error': magnitude_error, + 'mag_band': band_label, + 'mag_column': mag_column, + 'mag_error_column': error_column, + 'uses_relaxed_bv_error_limit': bool(magnitude_error > base_error_limit), + } + return None + + +def sky_separation_arcsec(ra_a, dec_a, ra_b, dec_b): + ra_a_rad = radians(float(ra_a)) + dec_a_rad = radians(float(dec_a)) + ra_b_rad = radians(float(ra_b)) + dec_b_rad = radians(float(dec_b)) + delta_ra = ra_b_rad - ra_a_rad + delta_dec = dec_b_rad - dec_a_rad + haversine = ( + sin(delta_dec / 2.0) ** 2 + + cos(dec_a_rad) * cos(dec_b_rad) * sin(delta_ra / 2.0) ** 2 + ) + haversine = min(max(haversine, 0.0), 1.0) + separation_rad = 2.0 * atan2(sqrt(haversine), sqrt(1.0 - haversine)) + return float(separation_rad * 206264.80624709636) + + +def nextastro_photometry_catalog_match(catalog_response, ra, dec, obs_filter, + max_separation_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC, + max_magnitude_error=CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX): + effective_max_separation_arcsec = _finite_float(max_separation_arcsec) + if effective_max_separation_arcsec is None or effective_max_separation_arcsec <= 0: + effective_max_separation_arcsec = NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC + band_candidates = nextastro_photometry_band_candidates(obs_filter) + matches = [] + for row in nextastro_catalog_rows(catalog_response): + row_ra = _finite_float(row.get('ra')) + row_dec = _finite_float(row.get('dec')) + if row_ra is None or row_dec is None: + continue + magnitude = row_nextastro_magnitude(row, band_candidates, max_error=max_magnitude_error) + if magnitude is None: + continue + separation = sky_separation_arcsec(ra, dec, row_ra, row_dec) + if separation > effective_max_separation_arcsec: + continue + matches.append({ + **magnitude, + 'catalog_ra': row_ra, + 'catalog_dec': row_dec, + 'source_id': row.get('source_id'), + 'id': row.get('id'), + 'separation_arcsec': separation, + 'catalog_row': row, + }) + + if not matches: + return None + matches.sort(key=lambda match: ( + match['priority'], + match.get('magnitude_error_tier', 0), + match['separation_arcsec'], + )) + return matches[0] + + +@retry( + stop=stop_after_attempt(NEXTASTRO_VARIABILITY_MAX_RETRY_ATTEMPTS), + wait=wait_fixed(NEXTASTRO_VARIABILITY_RETRY_WAIT_SECONDS), + retry=retry_if_exception(should_retry_nextastro_variability_error), +) +def nextastro_photometry_cone_query(ra, dec, radius_arcsec, columns=None): + api_url = f'{NEXTASTRO_PHOTOMETRY_API_URL}/cone_query' + payload = { + 'columns': list(columns or NEXTASTRO_PHOTOMETRY_COLUMNS), + 'ra': float(ra), + 'dec': float(dec), + 'radius_arcsec': float(radius_arcsec), + } + log_info(f"NextAstro photometry catalog request JSON: {json.dumps(payload)}") + result = requests.post(api_url, json=payload, timeout=30) + if result.status_code != 200: + raise RuntimeError(f"NextAstro photometry catalog returned HTTP {result.status_code}.") + + body = result.json() + if not isinstance(body, dict) or not isinstance(body.get('rows'), list): + raise RuntimeError("NextAstro photometry catalog returned an unexpected response format.") + log_info( + "NextAstro photometry catalog response JSON: " + f"{json.dumps({'count': body.get('count'), 'columns': body.get('columns')})}" + ) + return body + + +def _validate_nextastro_object_match(result, context): + if not isinstance(result, dict): + raise RuntimeError( + f"NextAstro photometry catalog returned an invalid {context} result." + ) + match = result.get('match') + separation = result.get('separation_arcsec') + if match is not None and not isinstance(match, dict): + raise RuntimeError( + f"NextAstro photometry catalog returned an invalid {context} match." + ) + if match is None: + if separation is not None: + raise RuntimeError( + f"NextAstro photometry catalog returned a separation without a {context} match." + ) + return + if _finite_float(separation) is None or float(separation) < 0: + raise RuntimeError( + f"NextAstro photometry catalog returned an invalid {context} separation." + ) + + +@retry( + stop=stop_after_attempt(NEXTASTRO_VARIABILITY_MAX_RETRY_ATTEMPTS), + wait=wait_fixed(NEXTASTRO_VARIABILITY_RETRY_WAIT_SECONDS), + retry=retry_if_exception(should_retry_nextastro_variability_error), +) +def nextastro_photometry_single_object_query( + ra, dec, radius_arcsec, columns=None, required_columns=None): + payload = { + 'columns': list(columns or NEXTASTRO_PHOTOMETRY_COLUMNS), + 'ra': float(ra), + 'dec': float(dec), + 'radius_arcsec': float(radius_arcsec), + } + if required_columns: + payload['required_columns'] = list(required_columns) + log_info(f"NextAstro single-object photometry request JSON: {json.dumps(payload)}") + result = requests.post( + NEXTASTRO_PHOTOMETRY_SINGLE_OBJECT_URL, + json=payload, + timeout=30, + ) + if result.status_code != 200: + raise RuntimeError( + f"NextAstro single-object photometry lookup returned HTTP {result.status_code}." + ) + + body = result.json() + if not isinstance(body, dict) or not isinstance(body.get('columns'), list): + raise RuntimeError( + "NextAstro single-object photometry lookup returned an unexpected response format." + ) + _validate_nextastro_object_match(body, 'single-object') + log_info( + "NextAstro single-object photometry response JSON: " + f"{json.dumps({'matched': body.get('match') is not None, 'columns': body.get('columns')})}" + ) + return body + + +@retry( + stop=stop_after_attempt(NEXTASTRO_VARIABILITY_MAX_RETRY_ATTEMPTS), + wait=wait_fixed(NEXTASTRO_VARIABILITY_RETRY_WAIT_SECONDS), + retry=retry_if_exception(should_retry_nextastro_variability_error), +) +def nextastro_photometry_objects_query( + coordinates, radius_arcsec, columns=None, required_columns=None): + objects = [ + {'key': str(index), 'ra': float(ra), 'dec': float(dec)} + for index, (ra, dec) in enumerate(coordinates) + ] + if not objects: + return { + 'columns': list(columns or NEXTASTRO_PHOTOMETRY_COLUMNS), + 'count': 0, + 'results': [], + } + payload = { + 'columns': list(columns or NEXTASTRO_PHOTOMETRY_COLUMNS), + 'objects': objects, + 'radius_arcsec': float(radius_arcsec), + } + if required_columns: + payload['required_columns'] = list(required_columns) + log_info( + "NextAstro multi-object photometry request JSON: " + f"{json.dumps({'objects': objects, 'radius_arcsec': payload['radius_arcsec']})}" + ) + result = requests.post( + NEXTASTRO_PHOTOMETRY_OBJECTS_QUERY_URL, + json=payload, + timeout=30, + ) + if result.status_code != 200: + raise RuntimeError( + f"NextAstro multi-object photometry lookup returned HTTP {result.status_code}." + ) + + body = result.json() + results = body.get('results') if isinstance(body, dict) else None + if ( + not isinstance(body, dict) + or not isinstance(body.get('columns'), list) + or not isinstance(body.get('count'), int) + or not isinstance(results, list) + or len(results) != len(objects) + ): + raise RuntimeError( + "NextAstro multi-object photometry lookup returned an unexpected response format." + ) + for index, (requested, object_result) in enumerate(zip(objects, results)): + _validate_nextastro_object_match(object_result, f'multi-object #{index + 1}') + if object_result.get('key') != requested['key']: + raise RuntimeError( + "NextAstro multi-object photometry lookup returned results out of order." + ) + response_ra = _finite_float(object_result.get('ra')) + response_dec = _finite_float(object_result.get('dec')) + if ( + response_ra is None + or response_dec is None + or sky_separation_arcsec( + requested['ra'], requested['dec'], response_ra, response_dec + ) > 0.01 + ): + raise RuntimeError( + "NextAstro multi-object photometry lookup returned mismatched coordinates." + ) + matched_count = sum(item.get('match') is not None for item in results) + if body['count'] != matched_count: + raise RuntimeError( + "NextAstro multi-object photometry lookup returned an inconsistent match count." + ) + log_info( + "NextAstro multi-object photometry response JSON: " + f"{json.dumps({'count': body['count'], 'requested': len(objects), 'columns': body['columns']})}" + ) + return body + + +def nextastro_photometry_match_from_object_result( + object_result, + ra, + dec, + obs_filter, + radius_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): + if not isinstance(object_result, dict) or object_result.get('match') is None: + return None + return nextastro_photometry_catalog_match( + { + 'row_format': 'objects', + 'rows': [object_result['match']], + }, + ra, + dec, + obs_filter, + max_separation_arcsec=radius_arcsec, + ) + + +def nextastro_photometry_for_coordinates( + coordinates, + obs_filter, + radius_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): + coordinates = list(coordinates) + if not coordinates: + return [] + lookup_columns = nextastro_photometry_lookup_columns(obs_filter) + if lookup_columns is None: + return [None] * len(coordinates) + if len(coordinates) == 1: + ra, dec = coordinates[0] + return [ + nextastro_photometry_for_coordinate( + ra, + dec, + obs_filter, + radius_arcsec=radius_arcsec, + ) + ] + response = nextastro_photometry_objects_query( + coordinates, + radius_arcsec, + **lookup_columns, + ) + return [ + nextastro_photometry_match_from_object_result( + object_result, + ra, + dec, + obs_filter, + radius_arcsec=radius_arcsec, + ) + for (ra, dec), object_result in zip(coordinates, response['results']) + ] + + +def nextastro_photometry_catalog_for_wcs(wcs_file, axis, img_scale, obs_filter): + if not wcs_file or img_scale is None: + return None + image_width, image_height = float(axis[0]), float(axis[1]) + if not (np.isfinite(image_width) and np.isfinite(image_height) and np.isfinite(float(img_scale))): + return None + + wcs_hdr = search_wcs(wcs_file) + center_ra, center_dec = wcs_hdr.pixel_to_world_values(image_width / 2.0, image_height / 2.0) + radius_arcsec = 0.5 * float(img_scale) * float(np.hypot(image_width, image_height)) + radius_arcsec += NEXTASTRO_PHOTOMETRY_FIELD_PADDING_ARCSEC + log_info( + "\nQuerying NextAstro photometry catalog for the full reduced field " + f"(radius={radius_arcsec:.1f} arcsec)." + ) + return nextastro_photometry_cone_query(center_ra, center_dec, radius_arcsec) + + +def nextastro_photometry_for_coordinate(ra, dec, obs_filter, + radius_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): + lookup_columns = nextastro_photometry_lookup_columns(obs_filter) + if lookup_columns is None: + return None + object_result = nextastro_photometry_single_object_query( + ra, + dec, + radius_arcsec, + **lookup_columns, + ) + return nextastro_photometry_match_from_object_result( + object_result, + ra, + dec, + obs_filter, + radius_arcsec=radius_arcsec, + ) + + +def nextastro_calibration_label(match): + source_id = match.get('source_id') or match.get('id') + if source_id not in (None, ''): + return f"NextAstro-{source_id}" + return f"RA{match['ra']:.6f}_DEC{match['dec']:.6f}" + + +def unique_nextastro_calibration_label(calibration_stars, match): + label = nextastro_calibration_label(match) + unique_label = label + duplicate_index = 2 + while unique_label in calibration_stars: + unique_label = f"{label}-{duplicate_index}" + duplicate_index += 1 + return unique_label + + +def estimate_target_pixel_from_ra_dec(info_dict, wcs_header, image_data, obs_time, + centroid_margin=7.5): + target_ra, target_dec = update_coordinates_with_proper_motion(info_dict, obs_time) + try: + x_pixel, y_pixel = project_ra_dec_to_wcs_pixel(target_ra, target_dec, wcs_header) + except Exception as exc: + log_info( + "Warning: Could not project target RA/Dec onto the new reference image " + f"({exc}).", + warn=True, + ) + return None + + if not pixel_within_image(x_pixel, y_pixel, image_data.shape): + log_info( + "Warning: target RA/Dec projects outside the new reference image; " + "the rejected-reference fallback cannot re-estimate target pixels.", + warn=True, + ) + return None + + centroid_x, centroid_y, sigma_x, sigma_y = np.nan, np.nan, np.nan, np.nan + if pixel_within_image(x_pixel, y_pixel, image_data.shape, margin=centroid_margin): + centroid_x, centroid_y, sigma_x, sigma_y = get_psf_parameters(image_data, x_pixel, y_pixel) + + if np.isfinite(centroid_x) and np.isfinite(centroid_y): + log_info( + "Reference fallback target position: " + f"RA={float(target_ra):.7f}, Dec={float(target_dec):.7f} projected to " + f"[{x_pixel:.2f}, {y_pixel:.2f}] and centroided to " + f"[{centroid_x:.2f}, {centroid_y:.2f}].", + warn=True, + ) + return float(centroid_x), float(centroid_y), float(target_ra), float(target_dec) + + log_info( + "Reference fallback target position: " + f"RA={float(target_ra):.7f}, Dec={float(target_dec):.7f} projected to " + f"[{x_pixel:.2f}, {y_pixel:.2f}]; centroid fit was unavailable, so the WCS-projected " + "pixel position will be used.", + warn=True, + ) + return x_pixel, y_pixel, float(target_ra), float(target_dec) + + +def connected_component_sizes(mask): + mask = np.asarray(mask, dtype=bool) + if mask.ndim != 2 or not np.any(mask): + return None, [] + + # Eight-connected labeling is equivalent to the former Python flood fill, + # but runs in compiled scipy code and avoids visiting every threshold pixel + # through nested Python loops on multi-megapixel reference frames. + labels, component_count = ndimage_label( + mask, + structure=np.ones((3, 3), dtype=np.uint8), + ) + if component_count <= 0: + return None, [] + component_sizes = np.bincount(labels.reshape(-1), minlength=component_count + 1)[1:] + labels = labels.astype(np.int32, copy=False) + labels -= 1 # Preserve the existing API: background=-1, components start at zero. + return labels, component_sizes.tolist() + + +def detect_reference_fallback_bright_stars( + image_data, + max_stars=REFERENCE_FALLBACK_DETECTION_MAX_STARS, + min_sep=REFERENCE_FALLBACK_DETECTION_MIN_SEP_PIXELS, + aperture_radius=REFERENCE_FALLBACK_DETECTION_APERTURE_RADIUS_PIXELS, + min_area=REFERENCE_FALLBACK_DETECTION_MIN_AREA_PIXELS, + threshold_percentile=99.7): + if image_data is None: + return [] + + data = np.array(image_data, dtype=np.float64, copy=True) + if data.ndim != 2: + return [] + + finite = np.isfinite(data) + if not finite.any(): + return [] + + median = float(np.nanmedian(data[finite])) + data[~finite] = median + signal = data - median + signal[signal < 0] = 0 + if not np.any(signal > 0): + return [] + + try: + threshold_percentile = float(threshold_percentile) + except (TypeError, ValueError): + threshold_percentile = 99.7 + threshold_percentile = min(max(threshold_percentile, 0.0), 100.0) + threshold = float(np.percentile(signal, threshold_percentile)) + if threshold <= 0: + threshold = float(np.percentile(signal, 99.0)) + if threshold <= 0: + positive_signal = signal[signal > 0] + if positive_signal.size == 0: + return [] + threshold = float(np.nanmin(positive_signal)) + + height, width = signal.shape + margin = max(int(min_sep), int(aperture_radius) + 2) + threshold_mask = signal >= threshold + threshold_flat_indices = np.flatnonzero(threshold_mask) + if threshold_flat_indices.size == 0: + return [] + threshold_values = signal.reshape(-1)[threshold_flat_indices] + flat_order = threshold_flat_indices[np.argsort(threshold_values)[::-1]] + source_labels, source_sizes = connected_component_sizes(threshold_mask) + stars = [] + separation_excluded = np.zeros(signal.shape, dtype=bool) + separation_radius = float(min_sep) + + for flat_index in flat_order: + y_pos, x_pos = np.unravel_index(int(flat_index), signal.shape) + peak = float(signal[y_pos, x_pos]) + if peak < threshold: + break + if source_labels is not None: + component_id = int(source_labels[y_pos, x_pos]) + if component_id < 0: + continue + if int(source_sizes[component_id]) < max(int(min_area), 1): + continue + if x_pos < margin or y_pos < margin or x_pos >= (width - margin) or y_pos >= (height - margin): + continue + if separation_excluded[y_pos, x_pos]: + continue + + radius = float(aperture_radius) + x_min = max(0, int(np.floor(x_pos - radius))) + x_max = min(width, int(np.ceil(x_pos + radius)) + 1) + y_min = max(0, int(np.floor(y_pos - radius))) + y_max = min(height, int(np.ceil(y_pos + radius)) + 1) + local_signal = signal[y_min:y_max, x_min:x_max] + local_y, local_x = np.ogrid[y_min:y_max, x_min:x_max] + local_aperture = ( + (local_x - x_pos) ** 2 + (local_y - y_pos) ** 2 + <= radius ** 2 + ) + flux = float(local_signal[local_aperture].sum()) + if flux <= 0: + continue + stars.append({'x': float(x_pos), 'y': float(y_pos), 'flux': flux}) + exclusion_x_min = max(0, int(np.floor(x_pos - separation_radius))) + exclusion_x_max = min(width, int(np.ceil(x_pos + separation_radius)) + 1) + exclusion_y_min = max(0, int(np.floor(y_pos - separation_radius))) + exclusion_y_max = min(height, int(np.ceil(y_pos + separation_radius)) + 1) + exclusion_y, exclusion_x = np.ogrid[ + exclusion_y_min:exclusion_y_max, + exclusion_x_min:exclusion_x_max, + ] + exclusion_circle = ( + (exclusion_x - x_pos) ** 2 + (exclusion_y - y_pos) ** 2 + < separation_radius ** 2 + ) + separation_excluded[ + exclusion_y_min:exclusion_y_max, + exclusion_x_min:exclusion_x_max, + ][exclusion_circle] = True + if len(stars) >= max_stars: + break + + stars.sort(key=lambda star: star['flux'], reverse=True) + return stars + + +def dedupe_reference_fallback_stars(stars, dedupe_radius=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS): + deduped_stars = [] + for star in sorted(stars or [], key=lambda value: float(value.get('flux', 0.0)), reverse=True): + x_pos = _finite_float(star.get('x')) + y_pos = _finite_float(star.get('y')) + if x_pos is None or y_pos is None: + continue + if any( + abs(float(existing.get('x', 0.0)) - x_pos) <= dedupe_radius + and abs(float(existing.get('y', 0.0)) - y_pos) <= dedupe_radius + for existing in deduped_stars + ): + continue + normalized = dict(star) + normalized['x'] = float(x_pos) + normalized['y'] = float(y_pos) + flux = _finite_float(star.get('flux')) + if flux is not None: + normalized['flux'] = float(flux) + deduped_stars.append(normalized) + return deduped_stars + + +def filter_reference_fallback_stars_to_middle_fifty_percent(stars, image_shape): + try: + height, width = image_shape[:2] + width = float(width) + height = float(height) + except Exception: + return list(stars or []) + + if width <= 1.0 or height <= 1.0: + return list(stars or []) + + center_x = (width - 1.0) / 2.0 + center_y = (height - 1.0) / 2.0 + half_width = (width - 1.0) * 0.25 + half_height = (height - 1.0) * 0.25 + + filtered = [] + for star in stars or []: + x_pos = _finite_float(star.get('x')) + y_pos = _finite_float(star.get('y')) + if x_pos is None or y_pos is None: + continue + if abs(x_pos - center_x) > half_width or abs(y_pos - center_y) > half_height: + continue + filtered.append(star) + return filtered + + +def nearest_reference_fallback_star_by_pixels(stars, x_value, y_value, max_sep_pixels=None, used_ids=None): + target_x = _finite_float(x_value) + target_y = _finite_float(y_value) + if target_x is None or target_y is None: + return None + + best_star = None + best_dist2 = None + for star in stars or []: + if used_ids and id(star) in used_ids: + continue + star_x = _finite_float(star.get('x')) + star_y = _finite_float(star.get('y')) + if star_x is None or star_y is None: + continue + dist2 = ((star_x - target_x) ** 2) + ((star_y - target_y) ** 2) + if best_dist2 is None or dist2 < best_dist2: + best_star = star + best_dist2 = dist2 + + if best_star is None: + return None + if max_sep_pixels is not None and best_dist2 is not None and best_dist2 > (float(max_sep_pixels) ** 2): + return None + return best_star + + +def select_reference_fallback_comparison_stars( + image_data, + image_shape, + target_pixel, + comp_count=REFERENCE_FALLBACK_COMPARISON_LIMIT, + min_comp_target_sep=REFERENCE_FALLBACK_MIN_COMP_TARGET_SEP_PIXELS): + max_count = max(0, int(comp_count)) + if max_count == 0 or image_data is None or image_shape is None: + return [], [] + + target_pixel = np.asarray(target_pixel, dtype=float).reshape(-1) + if target_pixel.size < 2 or not np.all(np.isfinite(target_pixel[:2])): + return [], [] + target_x, target_y = float(target_pixel[0]), float(target_pixel[1]) + + stars = detect_reference_fallback_bright_stars(image_data) + comp_pool = filter_reference_fallback_stars_to_middle_fifty_percent( + dedupe_reference_fallback_stars(stars), + image_shape, + ) + detected_target = nearest_reference_fallback_star_by_pixels( + comp_pool, + target_x, + target_y, + max_sep_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) + + used_ids = set() + if detected_target is not None: + used_ids.add(id(detected_target)) + + comp_candidates = [] + min_sep2 = max(float(min_comp_target_sep), 0.0) ** 2 + for star in comp_pool: + if id(star) in used_ids: + continue + dx = float(star.get('x', 0.0)) - target_x + dy = float(star.get('y', 0.0)) - target_y + if min_sep2 > 0.0 and ((dx * dx) + (dy * dy)) < min_sep2: + continue + comp_candidates.append(star) + if len(comp_candidates) >= max_count: + break + + comp_stars = [[float(star['x']), float(star['y'])] for star in comp_candidates] + return comp_stars, comp_candidates + + +def log_reference_fallback_comparison_candidates(comp_stars, detected_candidates): + if not comp_stars: + log_info( + "Warning: the nextastro_archive-style bright-star picker did not find any usable replacement " + "comparison stars on the new reference image.", + warn=True, + ) + return + + log_info( + f"Reference fallback replaced supplied comparison-star pixels with {len(comp_stars)} " + "image-detected bright-star candidate(s) selected like nextastro_archive " + "(central 50% of the frame, de-duplicated detections, at least " + f"{REFERENCE_FALLBACK_MIN_COMP_TARGET_SEP_PIXELS:g} px from the target, brightest-first).", + warn=True, + ) + for index, star in enumerate(detected_candidates, start=1): + log_info( + f"Reference fallback comparison candidate #{index}: " + f"pixels=[{float(star['x']):.2f}, {float(star['y']):.2f}], " + f"aperture flux={float(star.get('flux', np.nan)):.3g}.", + warn=True, + ) + + +def image_aperture_signal_flux(image_data, x_pos, y_pos, + aperture_radius=REFERENCE_FALLBACK_DETECTION_APERTURE_RADIUS_PIXELS): + if image_data is None: + return np.nan + data = np.asarray(image_data, dtype=float) + if data.ndim != 2: + return np.nan + x_pos = _finite_float(x_pos) + y_pos = _finite_float(y_pos) + if x_pos is None or y_pos is None: + return np.nan + + height, width = data.shape + radius = float(aperture_radius) + x_min = max(0, int(np.floor(x_pos - radius))) + x_max = min(width, int(np.ceil(x_pos + radius)) + 1) + y_min = max(0, int(np.floor(y_pos - radius))) + y_max = min(height, int(np.ceil(y_pos + radius)) + 1) + if x_min >= x_max or y_min >= y_max: + return np.nan + + finite = np.isfinite(data) + if not np.any(finite): + return np.nan + background = float(np.nanmedian(data[finite])) + cutout = data[y_min:y_max, x_min:x_max] + local_y, local_x = np.ogrid[y_min:y_max, x_min:x_max] + mask = (local_x - x_pos) ** 2 + (local_y - y_pos) ** 2 <= radius ** 2 + aperture_values = cutout[mask] + aperture_values = aperture_values[np.isfinite(aperture_values)] + if aperture_values.size == 0: + return np.nan + flux = float(np.sum(aperture_values - background)) + return flux if np.isfinite(flux) and flux > 0 else np.nan + + +def nextastro_color_candidate_pairs(obs_filter): + filter_key = normalize_nextastro_filter_key(obs_filter) + if observed_filter_uses_clear_v_calibration(obs_filter): + preferred = [('Bmag', 'Vmag', 'B-V')] + elif filter_key in ('u', 'johnsonu', 'su', 'up'): + preferred = [('umag', 'g', 'u-g')] + elif filter_key in ('b', 'johnsonb', 'photographicb', 'bb', 'pb'): + preferred = [('Bmag', 'Vmag', 'B-V')] + elif filter_key in ('v', 'johnsonv', 'bv', 'cv', 'clearv', 'c', 'clear', 'lum', 'luminance'): + preferred = [('Bmag', 'Vmag', 'B-V')] + elif filter_key in ('sg', 'sloang', 'sdssg', 'gp', 'g', 'tg'): + preferred = [('g', 'r', 'g-r')] + elif filter_key in ('sr', 'sloanr', 'sdssr', 'johnsonr', 'cousinsr', 'rp', 'r', 'rc', 'rj', 'pr', 'tr', 'cr'): + preferred = [('r', 'i', 'r-i')] + elif filter_key in ('si', 'sloani', 'sdssi', 'johnsoni', 'cousinsi', 'ip', 'i', 'ic', 'ij'): + preferred = [('r', 'i', 'r-i')] + elif filter_key in ('sz', 'sloanz', 'sdssz', 'zp', 'z', 'zs'): + preferred = [('i', 'z', 'i-z')] + else: + preferred = [] + + universal_fallbacks = [ + ('Bmag', 'Vmag', 'B-V'), + ('phot_bp_mean_mag', 'phot_rp_mean_mag', 'BP-RP'), + ] + return preferred + [pair for pair in universal_fallbacks if pair not in preferred] + + +def nextastro_catalog_bp_rp(row): + for direct_key in ( + 'bp_rp', 'BP_RP', 'BP-RP', 'BPRP', 'gaia_bp_rp', 'GAIA_BP_RP', 'phot_bp_rp' + ): + direct_value = _finite_float(row.get(direct_key)) + if direct_value is not None: + return direct_value, direct_key, None + + for bp_key, rp_key in ( + ('phot_bp_mean_mag', 'phot_rp_mean_mag'), + ('PHOT_BP_MEAN_MAG', 'PHOT_RP_MEAN_MAG'), + ('GAIA_BP', 'GAIA_RP'), + ('BP_MAG', 'RP_MAG'), + ('BP', 'RP'), + ): + bp_magnitude = _finite_float(row.get(bp_key)) + rp_magnitude = _finite_float(row.get(rp_key)) + if bp_magnitude is not None and rp_magnitude is not None: + return float(bp_magnitude - rp_magnitude), bp_key, rp_key + return None + + +@lru_cache(maxsize=2048) +def _cached_nextastro_gaia_bp_rp(ra, dec, max_separation_arcsec): + response = requests.get( + NEXTASTRO_GAIA_DISTPM_ENDPOINT, + params={'ra': float(ra), 'dec': float(dec)}, + timeout=NEXTASTRO_GAIA_COLOR_LOOKUP_TIMEOUT_SECONDS, + ) + if response.status_code != 200: + raise RuntimeError(f"NextAstro Gaia lookup returned HTTP {response.status_code}.") + body = response.json() + gaia = body.get('gaia') if isinstance(body, dict) else None + if not isinstance(gaia, dict): + return None + separation = _finite_float(gaia.get('separation_arcsec')) + if separation is None or separation > float(max_separation_arcsec): + return None + bp_rp = nextastro_catalog_bp_rp(gaia) + if bp_rp is None: + return None + color, first_column, second_column = bp_rp + return { + 'color': float(color), + 'label': 'BP-RP', + 'first_column': first_column, + 'second_column': second_column, + 'catalog_source': 'NextAstro Gaia DR3', + 'gaia_source_id': gaia.get('source_id'), + 'gaia_separation_arcsec': separation, + } + + +def nextastro_gaia_bp_rp_for_coordinate( + ra, dec, max_separation_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): + parsed_ra = _finite_float(ra) + parsed_dec = _finite_float(dec) + parsed_radius = _finite_float(max_separation_arcsec) + if parsed_ra is None or parsed_dec is None or parsed_radius is None or parsed_radius <= 0: + return None + return _cached_nextastro_gaia_bp_rp( + round(parsed_ra, 7), + round(parsed_dec, 7), + round(parsed_radius, 3), + ) + + +def nextastro_catalog_color_from_pairs(row, pairs): + if not isinstance(row, dict): + return None + for first_column, second_column, label in pairs: + if label == 'BP-RP': + bp_rp = nextastro_catalog_bp_rp(row) + if bp_rp is None: + continue + color, first_column, second_column = bp_rp + return { + 'color': float(color), + 'label': label, + 'first_column': first_column, + 'second_column': second_column, + } + first = _finite_float(row.get(first_column)) + second = _finite_float(row.get(second_column)) + if first is None or second is None: + continue + return { + 'color': float(first - second), + 'label': label, + 'first_column': first_column, + 'second_column': second_column, + } + return None + + +def nextastro_catalog_color(row, obs_filter): + return nextastro_catalog_color_from_pairs( + row, + nextastro_color_candidate_pairs(obs_filter), + ) + + +def normalize_colour_index_label(value): + text = str(value or '').strip().upper().replace(' ', '') + aliases = { + 'B-V': 'B-V', + 'BV': 'B-V', + 'BP-RP': 'BP-RP', + 'BPRP': 'BP-RP', + 'GBP-GRP': 'BP-RP', + 'GAIABP-RP': 'BP-RP', + 'R-I': 'R-I', + 'RI': 'R-I', + 'G-R': 'G-R', + 'GR': 'G-R', + 'I-Z': 'I-Z', + 'IZ': 'I-Z', + 'U-G': 'U-G', + 'UG': 'U-G', + } + return aliases.get(text, text) + + +def colour_term_metadata_from_info(info_dict): + if not isinstance(info_dict, dict): + return {} + + def _number(*keys, nonnegative=False): + for key in keys: + value = _finite_float(info_dict.get(key)) + if value is None: + continue + if value <= -90.0: + continue + if nonnegative and value < 0.0: + continue + return float(value) + return None + + metadata = { + 'term': _number('colour_term', 'color_term', 'COLTERM'), + 'term_error': _number( + 'colour_term_error', + 'color_term_error', + 'COLTERR', + nonnegative=True, + ), + 'term_index': normalize_colour_index_label( + info_dict.get('colour_term_index') + or info_dict.get('color_term_index') + or info_dict.get('COLTIDX') + ), + 'bv_term': _number('colour_term_bv', 'color_term_bv', 'COLTBV'), + 'bv_error': _number( + 'colour_term_bv_error', + 'color_term_bv_error', + 'COLTBVER', + 'COLTBVERR', + nonnegative=True, + ), + 'bprp_term': _number('colour_term_bprp', 'color_term_bprp', 'COLTBPRP'), + 'bprp_error': _number( + 'colour_term_bprp_error', + 'color_term_bprp_error', + 'CBPRPERR', + 'COLTBPRPERR', + nonnegative=True, + ), + 'equation_filter': ( + info_dict.get('colour_equation_filter') + or info_dict.get('color_equation_filter') + or info_dict.get('COLEQFIL') + ), + } + return metadata + + +def colour_term_for_catalog_label(metadata, color_label): + if not isinstance(metadata, dict): + return None, None + normalized_label = normalize_colour_index_label(color_label) + if normalized_label == 'B-V': + term = metadata.get('bv_term') + error = metadata.get('bv_error') + if term is not None: + return term, error + if normalized_label == 'BP-RP': + term = metadata.get('bprp_term') + error = metadata.get('bprp_error') + if term is not None: + return term, error + + generic_term = metadata.get('term') + generic_index = normalize_colour_index_label(metadata.get('term_index')) + if generic_term is not None and (not generic_index or generic_index == normalized_label): + return generic_term, metadata.get('term_error') + if generic_term is not None and normalized_label not in {'B-V', 'BP-RP'}: + return generic_term, metadata.get('term_error') + return None, None + + +def nextastro_catalog_nearest_color_row(catalog_response, ra, dec, obs_filter, + max_separation_arcsec= + AUTOMATIC_CALIBRATION_SELECTOR_COLOR_MATCH_RADIUS_ARCSEC, + gaia_lookup_state=None, + gaia_match_radius_arcsec= + NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): + nearby_rows = [] + for row in nextastro_catalog_rows(catalog_response): + row_ra = _finite_float(row.get('ra')) + row_dec = _finite_float(row.get('dec')) + if row_ra is None or row_dec is None: + continue + separation = sky_separation_arcsec(ra, dec, row_ra, row_dec) + if separation > float(max_separation_arcsec): + continue + nearby_rows.append((separation, row, row_ra, row_dec)) + + for separation, row, row_ra, row_dec in sorted(nearby_rows, key=lambda item: item[0]): + color = nextastro_catalog_color_with_gaia_fallback( + row, + obs_filter, + lookup_state=gaia_lookup_state, + max_separation_arcsec=gaia_match_radius_arcsec, + ) + if color is None: + continue + return { + 'catalog_row': row, + 'catalog_ra': row_ra, + 'catalog_dec': row_dec, + 'source_id': row.get('source_id'), + 'id': row.get('id'), + 'separation_arcsec': separation, + 'color': color, + } + return None + + +def select_automatic_optimal_calibration_stars( + image_data, + image_shape, + target_pixel, + ra_wcs, + dec_wcs, + obs_filter, + field_catalog, + count=AUTOMATIC_CALIBRATION_SELECTOR_DEFAULT_COUNT, + min_comp_target_sep=REFERENCE_FALLBACK_MIN_COMP_TARGET_SEP_PIXELS, + colour_term_metadata=None, + brightest_first=False, + saturation_threshold=None, + catalog_match_radius_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): + max_count = parse_automatic_calibration_selector_count(count) + if image_data is None or field_catalog is None: + return [], [] + + target_pixel = np.asarray(target_pixel, dtype=float).reshape(-1) + if target_pixel.size < 2 or not np.all(np.isfinite(target_pixel[:2])): + return [], [] + target_x, target_y = float(target_pixel[0]), float(target_pixel[1]) + + target_flux = image_aperture_signal_flux(image_data, target_x, target_y) + if not np.isfinite(target_flux) or target_flux <= 0: + log_info( + "Warning: automatic calibration selector could not measure a positive target flux " + "on the reference image.", + warn=True, + ) + return [], [] + + height, width = image_shape[:2] + target_xi = int(np.clip(round(target_x), 0, width - 1)) + target_yi = int(np.clip(round(target_y), 0, height - 1)) + gaia_lookup_state = { + 'remaining': min( + NEXTASTRO_GAIA_COLOR_LOOKUP_MAX_PER_SELECTOR, + max(1, max_count * 2 + 1), + ), + 'attempted': 0, + 'matched': 0, + } + if brightest_first: + target_match = nextastro_photometry_catalog_match( + field_catalog, + ra_wcs[target_yi][target_xi], + dec_wcs[target_yi][target_xi], + obs_filter, + max_separation_arcsec=catalog_match_radius_arcsec, + ) + target_color = nextastro_catalog_color_with_gaia_fallback( + (target_match or {}).get('catalog_row'), + obs_filter, + lookup_state=gaia_lookup_state, + max_separation_arcsec=catalog_match_radius_arcsec, + ) + else: + target_match = nextastro_catalog_nearest_color_row( + field_catalog, + ra_wcs[target_yi][target_xi], + dec_wcs[target_yi][target_xi], + obs_filter, + gaia_lookup_state=gaia_lookup_state, + gaia_match_radius_arcsec=catalog_match_radius_arcsec, + ) + target_color = (target_match or {}).get('color') + if not brightest_first and target_color is None: + log_nextastro_gaia_color_lookup_summary(gaia_lookup_state) + log_info( + "Warning: automatic calibration selector could not derive a target color from the " + "NextAstro photometry catalog or Gaia DR3.", + warn=True, + ) + return [], [] + + log_info( + "Scanning the reference frame for bright, non-saturated stellar-variability " + "ensemble candidates." + ) + detection_started = perf_counter() + detected_stars = detect_reference_fallback_bright_stars( + image_data, + max_stars=max( + AUTOMATIC_CALIBRATION_SELECTOR_MAX_DETECTIONS, + max_count * 8, + ), + threshold_percentile=AUTOMATIC_CALIBRATION_SELECTOR_DETECTION_PERCENTILE, + ) + log_info( + "Reference-frame stellar-variability ensemble candidate scan found " + f"{len(detected_stars)} source(s) in {perf_counter() - detection_started:.2f} seconds." + ) + detected_pool = dedupe_reference_fallback_stars(detected_stars) + comp_pool = ( + detected_pool + if brightest_first + else filter_reference_fallback_stars_to_middle_fifty_percent(detected_pool, image_shape) + ) + target_detection = nearest_reference_fallback_star_by_pixels( + comp_pool, + target_x, + target_y, + max_sep_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) + used_ids = {id(target_detection)} if target_detection is not None else set() + + min_sep2 = max(float(min_comp_target_sep), 0.0) ** 2 + candidates = [] + for star in comp_pool: + if id(star) in used_ids: + continue + x_pos = _finite_float(star.get('x')) + y_pos = _finite_float(star.get('y')) + star_flux = _finite_float(star.get('flux')) + if x_pos is None or y_pos is None or star_flux is None: + continue + if not pixel_within_image(x_pos, y_pos, image_shape): + continue + if (x_pos - target_x) ** 2 + (y_pos - target_y) ** 2 < min_sep2: + continue + brightness_ratio = float(star_flux / target_flux) + if ( + not brightest_first + and not ( + AUTOMATIC_CALIBRATION_SELECTOR_BRIGHTNESS_MIN_RATIO + <= brightness_ratio + <= AUTOMATIC_CALIBRATION_SELECTOR_BRIGHTNESS_MAX_RATIO + ) + ): + continue + if ( + brightest_first + and _finite_float(saturation_threshold) is not None + and aperture_contains_overexposed_pixel( + image_data, + x_pos, + y_pos, + REFERENCE_FALLBACK_DETECTION_APERTURE_RADIUS_PIXELS, + float(saturation_threshold), + ) + ): + continue + xi = int(np.clip(round(x_pos), 0, width - 1)) + yi = int(np.clip(round(y_pos), 0, height - 1)) + comp_ra = _finite_float(ra_wcs[yi][xi]) + comp_dec = _finite_float(dec_wcs[yi][xi]) + if comp_ra is None or comp_dec is None: + continue + if brightest_first: + match = nextastro_photometry_catalog_match( + field_catalog, + comp_ra, + comp_dec, + obs_filter, + max_separation_arcsec=catalog_match_radius_arcsec, + ) + color = nextastro_catalog_color_with_gaia_fallback( + (match or {}).get('catalog_row'), + obs_filter, + lookup_state=gaia_lookup_state, + max_separation_arcsec=catalog_match_radius_arcsec, + ) + if ( + match is None + or catalog_band_priority(match.get('mag_band'), obs_filter) != 0 + ): + continue + color_delta = ( + abs(color['color'] - target_color['color']) + if color is not None and target_color is not None + else np.nan + ) + else: + match = nextastro_catalog_nearest_color_row( + field_catalog, + comp_ra, + comp_dec, + obs_filter, + gaia_lookup_state=gaia_lookup_state, + gaia_match_radius_arcsec=catalog_match_radius_arcsec, + ) + color = (match or {}).get('color') + if match is None or color is None: + continue + color_delta = abs(color['color'] - target_color['color']) + colour_term, colour_term_error = colour_term_for_catalog_label( + colour_term_metadata, + (color or {}).get('label'), + ) + expected_colour_mismatch_mag = None + colour_term_uncertainty_mag = None + if colour_term is not None and np.isfinite(colour_term) and np.isfinite(color_delta): + expected_colour_mismatch_mag = abs(float(colour_term)) * float(color_delta) + if colour_term_error is not None and np.isfinite(colour_term_error) and np.isfinite(color_delta): + colour_term_uncertainty_mag = abs(float(color_delta)) * float(colour_term_error) + candidates.append({ + 'x': float(x_pos), + 'y': float(y_pos), + 'flux': float(star_flux), + 'brightness_ratio': brightness_ratio, + 'ra': comp_ra, + 'dec': comp_dec, + 'catalog_match': match, + 'color': (color or {}).get('color', np.nan), + 'color_label': (color or {}).get('label', ''), + 'target_color': (target_color or {}).get('color', np.nan), + 'color_delta': float(color_delta), + 'catalog_magnitude': match.get('mag'), + 'catalog_magnitude_error': match.get('error'), + 'catalog_magnitude_band': match.get('mag_band'), + 'target_catalog_magnitude': (target_match or {}).get('mag'), + 'target_catalog_magnitude_error': (target_match or {}).get('error'), + 'target_catalog_magnitude_band': (target_match or {}).get('mag_band'), + 'colour_term': float(colour_term) if colour_term is not None else None, + 'colour_term_error': ( + float(colour_term_error) if colour_term_error is not None else None + ), + 'expected_colour_mismatch_mag': ( + float(expected_colour_mismatch_mag) + if expected_colour_mismatch_mag is not None + else None + ), + 'colour_term_uncertainty_mag': ( + float(colour_term_uncertainty_mag) + if colour_term_uncertainty_mag is not None + else None + ), + 'target_flux': float(target_flux), + }) + + if brightest_first: + candidates.sort( + key=lambda candidate: ( + -candidate['flux'], + _finite_float( + candidate.get('expected_colour_mismatch_mag'), + _finite_float(candidate.get('color_delta'), np.inf), + ), + ) + ) + else: + candidates.sort( + key=lambda candidate: ( + ( + candidate['expected_colour_mismatch_mag'] + if candidate.get('expected_colour_mismatch_mag') is not None + else candidate['color_delta'] + ), + abs(np.log(candidate['brightness_ratio'])), + -candidate['flux'], + ) + ) + selected_candidates = candidates[:max_count] + comp_stars = [[candidate['x'], candidate['y']] for candidate in selected_candidates] + log_nextastro_gaia_color_lookup_summary(gaia_lookup_state) + return comp_stars, selected_candidates + + +def log_automatic_optimal_calibration_selection(comp_stars, candidates, requested_count, + brightest_first=False): + if not comp_stars: + log_info( + "Warning: automatic optimal calibration selector did not find any usable comparison stars; " + "the existing comparison-star list will be kept.", + warn=True, + ) + return + if brightest_first: + candidate_text = "image-detected, non-saturated on the reference frame, and NextAstro matched" + ranking_text = "ranked brightest-first for the stellar-variability ensemble" + else: + candidate_text = "image-detected, flux-matched to 0.5-2.0x the target, and NextAstro matched" + ranking_text = "ranked by catalog color similarity to the target" + log_info( + "Automatic optimal calibration selector chose " + f"{len(comp_stars)} comparison star(s) out of the requested {requested_count}. " + f"Candidates were {candidate_text} and {ranking_text}." + ) + for index, candidate in enumerate(candidates, start=1): + if brightest_first: + catalog_band = candidate.get('catalog_magnitude_band') or 'magnitude' + log_info( + f" Stellar-variability ensemble candidate #{index}: " + f"pixels=[{candidate['x']:.2f}, {candidate['y']:.2f}], " + f"aperture flux={candidate['flux']:.3g}, " + f"catalog {catalog_band}=" + f"{candidate.get('catalog_magnitude', np.nan):.3f} +/- " + f"{candidate.get('catalog_magnitude_error', np.nan):.3f} mag." + ) + continue + mismatch_text = "" + if candidate.get('expected_colour_mismatch_mag') is not None: + mismatch_text = ( + f", expected_colour_mismatch={candidate['expected_colour_mismatch_mag']:.5f} mag" + ) + if candidate.get('colour_term_uncertainty_mag') is not None: + mismatch_text += ( + f", colour_term_sigma={candidate['colour_term_uncertainty_mag']:.5f} mag" + ) + log_info( + f" Auto comp #{index}: pixels=[{candidate['x']:.2f}, {candidate['y']:.2f}], " + f"flux_ratio={candidate['brightness_ratio']:.3f}, " + f"{candidate['color_label']}={candidate['color']:.3f}, " + f"target_{candidate['color_label']}={candidate['target_color']:.3f}, " + f"delta={candidate['color_delta']:.3f}{mismatch_text}." + ) + + +def calibration_catalog_identity(star, label=None): + if not isinstance(star, dict): + return None + catalog_row = star.get('catalog_row') if isinstance(star.get('catalog_row'), dict) else {} + source_id = star.get('source_id', catalog_row.get('source_id')) + if source_id not in (None, ''): + return 'source_id', str(source_id).strip() + catalog_id = star.get('id', catalog_row.get('id')) + if catalog_id not in (None, ''): + return 'catalog_id', str(catalog_id).strip() + catalog_ra = _finite_float(star.get('catalog_ra', catalog_row.get('ra'))) + catalog_dec = _finite_float(star.get('catalog_dec', catalog_row.get('dec'))) + if catalog_ra is not None and catalog_dec is not None: + return 'catalog_position', round(catalog_ra, 7), round(catalog_dec, 7) + if str(label or '').startswith('NextAstro-'): + return 'catalog_label', str(label) + return None + + +def nextastro_catalog_color_with_gaia_fallback( + row, + obs_filter, + lookup_state=None, + max_separation_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): + local_color = nextastro_catalog_color(row, obs_filter) + if local_color is not None or not isinstance(row, dict): + return local_color + if lookup_state is None or lookup_state.get('remaining', 0) <= 0: + return None + if lookup_state.get('error') is not None: + return None + + row_ra = _finite_float(row.get('ra')) + row_dec = _finite_float(row.get('dec')) + if row_ra is None or row_dec is None: + return None + + lookup_state['remaining'] -= 1 + lookup_state['attempted'] = lookup_state.get('attempted', 0) + 1 + try: + color = nextastro_gaia_bp_rp_for_coordinate( + row_ra, + row_dec, + max_separation_arcsec=max_separation_arcsec, + ) + except Exception as exc: + lookup_state['error'] = describe_retry_exception(exc) + return None + if color is not None: + lookup_state['matched'] = lookup_state.get('matched', 0) + 1 + return color + return None + + +def log_nextastro_gaia_color_lookup_summary(lookup_state): + if not isinstance(lookup_state, dict) or lookup_state.get('reported'): + return + lookup_state['reported'] = True + attempted = int(lookup_state.get('attempted', 0)) + matched = int(lookup_state.get('matched', 0)) + if attempted <= 0: + return + if matched: + log_info( + "Gaia DR3 BP-RP fallback supplied color data for " + f"{matched} of {attempted} queried star(s)." + ) + if lookup_state.get('error'): + log_info( + "Warning: Gaia DR3 BP-RP fallback became unavailable after " + f"{attempted} request(s): {lookup_state['error']}", + warn=True, + ) + elif lookup_state.get('remaining', 0) <= 0: + log_info( + "Warning: Gaia DR3 BP-RP fallback reached its per-selection request limit; " + "remaining candidates were evaluated only with photometry-catalog colors.", + warn=True, + ) + + +def merge_nextastro_calibration_stars( + comp_stars, + comp_ra_dec, + obs_filter, + existing_comp_stars=None, + field_catalog=None, + match_radius_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): + calibration_stars = dict(existing_comp_stars or {}) + existing_positions = { + tuple(value.get('pos', [])) + for value in calibration_stars.values() + if isinstance(value, dict) + } + existing_catalog_identities = { + identity + for label, value in calibration_stars.items() + if (identity := calibration_catalog_identity(value, label)) is not None + } + + candidates = [] + for index, (comp_pos, comp_radec) in enumerate(zip(comp_stars, comp_ra_dec)): + if tuple(comp_pos) in existing_positions: + continue + comp_ra = _finite_float(comp_radec[0]) + comp_dec = _finite_float(comp_radec[1]) + if comp_ra is None or comp_dec is None: + continue + + match = None + if field_catalog is not None: + match = nextastro_photometry_catalog_match( + field_catalog, + comp_ra, + comp_dec, + obs_filter, + max_separation_arcsec=match_radius_arcsec, + ) + candidates.append({ + 'index': index, + 'comp_pos': comp_pos, + 'ra': comp_ra, + 'dec': comp_dec, + 'match': match, + }) + + unresolved = [candidate for candidate in candidates if candidate['match'] is None] + remote_lookup_failed = False + if unresolved: + try: + remote_matches = nextastro_photometry_for_coordinates( + [(candidate['ra'], candidate['dec']) for candidate in unresolved], + obs_filter, + radius_arcsec=match_radius_arcsec, + ) + for candidate, match in zip(unresolved, remote_matches): + candidate['match'] = match + except Exception as exc: + remote_lookup_failed = True + log_info( + "Warning: NextAstro object photometry catalog lookup failed for " + f"{len(unresolved)} comparison star(s) ({describe_retry_exception(exc)}).", + warn=True, + ) + + added_count = 0 + for candidate in candidates: + index = candidate['index'] + comp_pos = candidate['comp_pos'] + comp_ra = candidate['ra'] + comp_dec = candidate['dec'] + match = candidate['match'] + if tuple(comp_pos) in existing_positions: + continue + if match is None: + if remote_lookup_failed: + continue + log_info( + f"Warning: NextAstro photometry catalog did not find a usable magnitude for " + f"comparison star #{index + 1}.", + warn=True, + ) + continue + if catalog_band_priority(match.get('mag_band'), obs_filter) != 0: + log_info( + "Warning: rejecting NextAstro photometry calibration for comparison star " + f"#{index + 1} because catalog band {match.get('mag_band')!r} does not match " + f"observed filter {obs_filter!r}.", + warn=True, + ) + continue + + catalog_identity = calibration_catalog_identity(match) + if catalog_identity is not None and catalog_identity in existing_catalog_identities: + log_info( + f"Skipping duplicate NextAstro calibration for comparison star #{index + 1}: " + "the same catalog source is already represented in the comparison pool." + ) + continue + + match.update({ + 'ra': comp_ra, + 'dec': comp_dec, + 'pos': list(comp_pos), + 'catalog_source': 'NextAstro photometry catalog', + 'is_aavso_vsp': False, + 'observed_filter': obs_filter, + }) + unique_label = unique_nextastro_calibration_label(calibration_stars, match) + calibration_stars[unique_label] = match + existing_positions.add(tuple(comp_pos)) + if catalog_identity is not None: + existing_catalog_identities.add(catalog_identity) + added_count += 1 + mag_text = magnitude_text(match['mag_band'], match['mag'], match['error']) + log_info( + f"NextAstro photometry calibration for comparison star #{index + 1}: " + f"{mag_text}, " + f"RA={comp_ra:.7f}, Dec={comp_dec:.7f}, " + f"catalog separation={match['separation_arcsec']:.2f} arcsec." + ) + + if added_count: + log_info(f"Added {added_count} NextAstro photometry catalog comparison star calibration(s).") + return calibration_stars + + +def nextastro_prereduced_calibration_star(phot_comp_star, obs_filter): + if not isinstance(phot_comp_star, dict): + return None, None + + comp_ra = _finite_float(phot_comp_star.get('ra')) + comp_dec = _finite_float(phot_comp_star.get('dec')) + if comp_ra is None or comp_dec is None: + return None, None + + match = nextastro_photometry_for_coordinate(comp_ra, comp_dec, obs_filter) + if match is None: + return None, None + if catalog_band_priority(match.get('mag_band'), obs_filter) != 0: + log_info( + "Warning: rejecting NextAstro photometry calibration for the pre-reduced " + f"comparison star because catalog band {match.get('mag_band')!r} does not match " + f"observed filter {obs_filter!r}.", + warn=True, + ) + return None, None + + match.update({ + 'ra': comp_ra, + 'dec': comp_dec, + 'pos': [ + phot_comp_star.get('x', ''), + phot_comp_star.get('y', ''), + ], + 'catalog_source': 'NextAstro photometry catalog', + 'is_aavso_vsp': False, + 'observed_filter': obs_filter, + }) + label = nextastro_calibration_label(match) + mag_text = magnitude_text(match['mag_band'], match['mag'], match['error']) + log_info( + "NextAstro photometry calibration for pre-reduced comparison star: " + f"{mag_text}, " + f"RA={comp_ra:.7f}, Dec={comp_dec:.7f}, " + f"catalog separation={match['separation_arcsec']:.2f} arcsec." + ) + return label, match + + +def check_for_variable_stars(ra_wcs, dec_wcs, comp_stars, use_nextastro_variability_server=False): + if use_nextastro_variability_server and comp_stars: + try: + log_info("\nChecking for variability using NextAstro variability server.") + comp_ra_dec = build_comp_ra_dec(ra_wcs, dec_wcs, comp_stars) + variability_flags = nextastro_variability_test(comp_ra_dec) + + for i, (comp_star, is_variable) in enumerate(zip(comp_stars[:], variability_flags)): + log_info(f"\nChecking for variability in Comparison Star #{i + 1}:" + f"\n\tPixel X: {comp_star[0]} Pixel Y: {comp_star[1]}" + f"\n\tNextAstro flagged variable: {is_variable}") + if is_variable: + comp_stars.remove(comp_star) + return + except Exception as e: + log_info(f"\nWarning: NextAstro variability server check failed ({describe_retry_exception(e)}). " + "Falling back to individual VSX variability checks.", warn=True) + + for i, comp_star in enumerate(comp_stars[:]): + ra = ra_wcs[int(comp_star[1])][int(comp_star[0])] + dec = dec_wcs[int(comp_star[1])][int(comp_star[0])] + + log_info(f"\nChecking for variability in Comparison Star #{i + 1}:" + f"\n\tPixel X: {comp_star[0]} Pixel Y: {comp_star[1]}") + if query_variable_star_apis(ra, dec): + comp_stars.remove(comp_star) + +# Apply calibrations if applicable +def apply_cals(image_data, gen_dark, gen_bias, gen_flat, i): + if gen_dark is not None and gen_dark.size != 0: + if i == 0: + log_info("Dark subtracting images.") + image_data = image_data - gen_dark + elif gen_bias is not None and gen_bias.size != 0: # if a dark is not available, then at least subtract off the pedestal via the bias + if i == 0: + log_info("Bias-correcting images.") + image_data = image_data - gen_bias + else: + pass + + if gen_flat is not None and gen_flat.size != 0: + if i == 0: + log_info("Flattening images.") + gen_flat[gen_flat == 0] = 1 + image_data = image_data / gen_flat + return image_data + +def calculate_demosaic_mult(demosaic_out): + if not demosaic_out: + return None + # Build vector to convert RBG pixels to single output + if isinstance(demosaic_out, list): + demosaic_mult = np.array(demosaic_out) + elif demosaic_out == 'red': + demosaic_mult = np.array([ 1.0, 0.0, 0.0 ]) + elif demosaic_out == 'green': + demosaic_mult = np.array([ 0.0, 1.0, 0.0 ]) + elif demosaic_out == 'blue': + demosaic_mult = np.array([ 0.0, 0.0, 1.0 ]) + elif demosaic_out == 'gray': + demosaic_mult = np.array([ 0.299, 0.587, 0.114 ]) # Same as rbg2gray + elif demosaic_out == 'blueblock': + demosaic_mult = np.array([ 0.299, 0.587, 0.0 ]) # drop blue, same mix of red, green as gray + else: # Green default + demosaic_mult = np.array([ 0.0, 1.0, 0.0 ]) + # Normalize + demosaic_mult = demosaic_mult / (demosaic_mult[0]+demosaic_mult[1]+demosaic_mult[2]) + return demosaic_mult + +# If demosaic requested, process +def demosaic_img(image_data, demosaic_fmt, demosaic_out, demosaic_mult, i): + if demosaic_fmt: + if i == 0: + log_info(f"Demosaicing images (mapping {demosaic_fmt} to {demosaic_out})") + img_dtype = image_data.dtype # Save data type + new_image_data = demosaicing_CFA_Bayer_bilinear(image_data, demosaic_fmt) + image_data = (new_image_data @ demosaic_mult).astype(img_dtype) + return image_data + +class AAVSOVSPUnavailableError(RuntimeError): + """Raised after the AAVSO VSP endpoint exhausts its response retries.""" + + +def fetch_aavso_vsp_chart(url): + """Fetch and validate a VSP chart, retrying transient/unusable responses.""" + total_attempts = AAVSO_VSP_MAX_RETRIES + 1 + for attempt_number in range(1, total_attempts + 1): + try: + response = requests.get(url, timeout=AAVSO_VSP_REQUEST_TIMEOUT_SECONDS) + response.raise_for_status() + data = response.json() + if not isinstance(data, dict): + raise ValueError( + f"AAVSO VSP returned {type(data).__name__} instead of a JSON object." + ) + missing_fields = [field for field in ('chartid', 'photometry') if field not in data] + if missing_fields: + raise ValueError( + "AAVSO VSP JSON response is missing required field(s): " + + ", ".join(missing_fields) + ) + if data['photometry'] is not None and not isinstance(data['photometry'], list): + raise ValueError("AAVSO VSP JSON field 'photometry' is not a list.") + return data + except (requests.RequestException, ValueError) as exc: + if attempt_number >= total_attempts: + raise AAVSOVSPUnavailableError( + f"AAVSO VSP returned no usable response after {total_attempts} attempts " + f"({AAVSO_VSP_MAX_RETRIES} retries): {describe_retry_exception(exc)}" + ) from exc + + retries_remaining = total_attempts - attempt_number + log_info( + "\nWarning: AAVSO VSP request failed " + f"on attempt {attempt_number}/{total_attempts} " + f"({describe_retry_exception(exc)}). Retrying in " + f"{AAVSO_VSP_RETRY_DELAY_SECONDS} seconds; " + f"{retries_remaining} retr{'y' if retries_remaining == 1 else 'ies'} remain.", + warn=True, + ) + sleep(AAVSO_VSP_RETRY_DELAY_SECONDS) + + +def vsp_query(file, axis, obs_filter, img_scale, maglimit=14, user_comp_stars=None, + user_targ_star=None, max_new_comp_stars=2): + if user_comp_stars is None: + user_comp_stars = [] + + try: + max_new_comp_stars = max(0, int(max_new_comp_stars)) + except (TypeError, ValueError): + max_new_comp_stars = 2 + + vsp_comp_stars_info = {} + vsp_star_count = 0 + observed_filter = obs_filter + + initial_user_comp_stars = [list(position) for position in user_comp_stars] + + wcs_hdr = search_wcs(file) + fov = (img_scale * max(axis)) / 60 + ra, dec = wcs_hdr.pixel_to_world_values(axis[0] // 2, axis[1] // 2) + # Respect limits from AAVSO API (as reported by API error messages) + if fov > 180 and maglimit > 12: + maglimit = 12 + + url = f"https://www.aavso.org/apps/vsp/api/chart/?format=json&ra={ra:5f}&dec={dec:5f}&fov={fov}&maglimit={maglimit}" + data = fetch_aavso_vsp_chart(url) + chart_id = data['chartid'] + + obs_filter = aavso_vsp_band_for_filter(obs_filter) + + vsp_candidates = [] + if data['photometry']: + for star in data['photometry']: + ra_deg, dec_deg = radec_hours_to_degree(star['ra'], star['dec']) + ra_pix, dec_pix = wcs_hdr.world_to_pixel_values(ra_deg, dec_deg) + + ra_pixel = float(np.asarray(ra_pix, dtype=float).reshape(-1)[0]) + dec_pixel = float(np.asarray(dec_pix, dtype=float).reshape(-1)[0]) + if not ( + 1 < ra_pixel < axis[0] + and 1 < dec_pixel < axis[1] + and obs_filter in [band['band'] for band in star['bands']] + ): + continue + star_info = next(band for band in star['bands'] if band['band'] == obs_filter) + usable_magnitude = usable_catalog_reference_magnitude( + star_info.get('mag'), + star_info.get('error'), + ) + if usable_magnitude is None: + continue + star_mag, star_mag_error = usable_magnitude + vsp_candidates.append({ + 'label': star['auid'], + 'pixel_position': [ra_pixel, dec_pixel], + 'star': { + 'mag': star_mag, + 'error': star_mag_error, + 'ra': ra_deg, + 'dec': dec_deg, + 'catalog_ra': ra_deg, + 'catalog_dec': dec_deg, + 'mag_band': obs_filter, + 'observed_filter': observed_filter, + 'catalog_source': 'AAVSO VSP', + 'is_aavso_vsp': True, + }, + }) + + # Match all supplied coordinates before applying the new-star cap. Greedy + # nearest-pair assignment makes the association one-to-one and prevents two + # nearby VSP sources from lending different magnitudes to the same measured star. + match_pairs = [] + for candidate_index, candidate in enumerate(vsp_candidates): + for user_index, user_position in enumerate(initial_user_comp_stars): + distance = comparison_star_pixel_distance( + candidate['pixel_position'], + user_position, + ) + if distance <= VSP_COMPARISON_MATCH_TOLERANCE_PIXELS: + match_pairs.append((distance, candidate_index, user_index)) + matched_candidates = {} + matched_user_indices = set() + for _, candidate_index, user_index in sorted(match_pairs): + if candidate_index in matched_candidates or user_index in matched_user_indices: + continue + matched_candidates[candidate_index] = list(initial_user_comp_stars[user_index]) + matched_user_indices.add(user_index) + + occupied_positions = [*initial_user_comp_stars] + if user_targ_star is not None: + occupied_positions.append(list(user_targ_star)) + matched_supplied_count = 0 + for candidate_index, candidate in enumerate(vsp_candidates): + candidate_position = candidate['pixel_position'] + if ( + user_targ_star is not None + and comparison_star_pixel_distance(candidate_position, user_targ_star) + <= VSP_COMPARISON_MATCH_TOLERANCE_PIXELS + ): + continue + + if candidate_index in matched_candidates: + vsp_star = matched_candidates[candidate_index] + matched_supplied_count += 1 + else: + # A candidate close to a supplied coordinate that was already assigned a + # nearer VSP source is ambiguous, so do not add it as a separate star. + if any( + comparison_star_pixel_distance(candidate_position, user_position) + <= VSP_COMPARISON_MATCH_TOLERANCE_PIXELS + for user_position in initial_user_comp_stars + ): + continue + if vsp_star_count >= max_new_comp_stars: + continue + vsp_star = [int(round(candidate_position[0])), int(round(candidate_position[1]))] + if any( + comparison_star_pixel_distance(vsp_star, occupied_position) + <= VSP_COMPARISON_MATCH_TOLERANCE_PIXELS + for occupied_position in occupied_positions + ): + continue + vsp_star_count = add_vsp_star(vsp_star_count, user_comp_stars, vsp_star) + occupied_positions.append(vsp_star) + + vsp_comp_stars_info[candidate['label']] = { + **candidate['star'], + 'pos': vsp_star, + } + + if not vsp_comp_stars_info: + log_info("\nNo comparison stars were gathered from AAVSO.\n") + if matched_supplied_count: + log_info( + f"\nMatched {matched_supplied_count} supplied comparison star coordinate(s) " + "one-to-one with AAVSO VSP photometry.\n" + ) + + return vsp_comp_stars_info, chart_id + + +def catalog_calibration_is_usable_for_filter(star, observed_filter, max_error=None): + if not isinstance(star, dict): + return False + magnitude = _finite_float(star.get('mag')) + magnitude_error = normalized_magnitude_error(star.get('error')) + if ( + not is_usable_apparent_magnitude(magnitude) + or magnitude_error is None + or catalog_band_priority(star.get('mag_band'), observed_filter) != 0 + ): + return False + effective_max_error = _finite_float(max_error) + return effective_max_error is None or magnitude_error <= effective_max_error + + +def merge_aavso_vsp_v_calibration_fallback( + file, axis, obs_filter, img_scale, calibration_stars, user_comp_stars, + user_targ_star=None, + max_new_comp_stars=STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS, + vsp_query_available=True): + """Query VSP when a V-family observation has no usable direct V calibration. + + Existing AAVSO VSP calibrations mean the field has already been queried. The + returned mapping contains the unified input-plus-VSP calibration pool, while + the second mapping contains only the VSP results from this fallback query. + """ + unified_calibrations = dict(calibration_stars or {}) + preferred_band = preferred_catalog_magnitude_band_for_filter(obs_filter) + if str(preferred_band or '').strip().upper() != 'V': + return unified_calibrations, {}, None, False + + usable_direct_v = any( + star.get('catalog_source') == 'NextAstro photometry catalog' + and catalog_calibration_is_usable_for_filter( + star, + obs_filter, + max_error=CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX, + ) + for star in unified_calibrations.values() + if isinstance(star, dict) + ) + if usable_direct_v: + return unified_calibrations, {}, None, False + + usable_vsp_v = any( + (star.get('is_aavso_vsp') or star.get('catalog_source') == 'AAVSO VSP') + and catalog_calibration_is_usable_for_filter( + star, + obs_filter, + max_error=CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX, + ) + for star in unified_calibrations.values() + if isinstance(star, dict) + ) + if usable_vsp_v: + return unified_calibrations, {}, None, False + + if not vsp_query_available: + log_info( + "Skipping the AAVSO VSP V-band calibration fallback because the earlier " + "VSP request already exhausted all retries.", + warn=True, + ) + return unified_calibrations, {}, None, False + + log_info( + "No usable direct V-band comparison calibration was returned by the NextAstro " + "photometry server; querying AAVSO VSP for this V-family observation." + ) + try: + vsp_calibrations, chart_id = vsp_query( + file, + axis, + obs_filter, + img_scale, + user_comp_stars=user_comp_stars, + user_targ_star=user_targ_star, + max_new_comp_stars=max_new_comp_stars, + ) + except Exception as exc: + log_info( + "Warning: automatic AAVSO VSP V-band calibration fallback failed " + f"({describe_retry_exception(exc)}).", + warn=True, + ) + return unified_calibrations, {}, None, True + + for label, star in vsp_calibrations.items(): + unified_calibrations[label] = star + if vsp_calibrations: + log_info( + f"Added {len(vsp_calibrations)} AAVSO VSP V-band calibration(s) to the " + "comparison-star calibration pool." + ) + return unified_calibrations, vsp_calibrations, chart_id, True + + +def add_vsp_star(vsp_star_count, user_comp_stars, vsp_star): + user_comp_stars.append(vsp_star) + log_info(f"\nAdded Comparison Star #{len(user_comp_stars)}, coordinates {vsp_star} from AAVSO") + + return vsp_star_count + 1 + + +def comparison_star_pixel_distance(first_position, second_position): + try: + first = np.asarray(first_position, dtype=float).reshape(-1) + second = np.asarray(second_position, dtype=float).reshape(-1) + except (TypeError, ValueError): + return np.inf + if first.size < 2 or second.size < 2 or not np.all(np.isfinite([*first[:2], *second[:2]])): + return np.inf + return float(np.hypot(first[0] - second[0], first[1] - second[1])) + + +def tracked_comparison_position(tracked_comparison_stars, comp_index): + """Return a position using the stable full tracking-list index space.""" + index = int(comp_index) + if index < 0 or index >= len(tracked_comparison_stars): + raise IndexError( + f"Tracked comparison index {index} is outside the " + f"{len(tracked_comparison_stars)}-star calibration pool." + ) + return list(tracked_comparison_stars[index]) + + +def selected_comparison_finder_entries(comparison_stars, comp_index=None, + ensemble_member_keys=None): + """Return labelled pixel positions for the selected single or ensemble reference.""" + + if ensemble_member_keys: + keys = list(ensemble_member_keys) + elif comp_index is not None: + keys = [f'comp{int(comp_index) + 1}'] + else: + keys = [] + + entries = [] + for key in keys: + match = re.fullmatch(r'comp(\d+)', str(key).strip(), flags=re.IGNORECASE) + if match is None: + continue + index = int(match.group(1)) - 1 + try: + position = tracked_comparison_position(comparison_stars, index) + position_values = np.asarray(position, dtype=float).reshape(-1) + except (IndexError, TypeError, ValueError): + continue + if position_values.size < 2 or not np.all(np.isfinite(position_values[:2])): + continue + entries.append({ + 'key': f'comp{index + 1}', + 'label': f'Comp {index + 1}', + 'position': [float(position_values[0]), float(position_values[1])], + }) + return entries + + +def check_comp_star_exists(user_stars, vsp_star, tol=VSP_COMPARISON_MATCH_TOLERANCE_PIXELS): + """Return the nearest user-entered comparison within ``tol`` pixels. + + Parameters + ---------- + user_stars : list + User-entered comparison-star pixel coordinates. + vsp_star : list + VSP star pixel coordinates. + tol : float + Maximum Euclidean pixel separation. + + Returns + ------- + bool + True if VSP star exists in user entered stars, otherwise False + list + The matching user coordinate, otherwise the original VSP coordinate. + """ + matches = [ + (comparison_star_pixel_distance(user_star, vsp_star), user_star) + for user_star in user_stars + ] + matches = [match for match in matches if match[0] <= float(tol)] + if matches: + return True, min(matches, key=lambda match: match[0])[1] + return False, vsp_star + + + +TRANSFORM_TIMING_STAGES = [ + 'astroalign_direct', + 'fft_translation', + 'astroalign_filtered', + 'astroalign_mask', + 'imreg_dft', +] + +_TRANSFORM_TIMING_STATS = { + stage: {'count': 0, 'success': 0, 'total_s': 0.0} for stage in TRANSFORM_TIMING_STAGES +} +_TRANSFORM_TIMING_STATS['mask_loops_skipped'] = 0 + +PHOTOMETRY_TIMING_STAGES = ['fit_centroid', 'aperPhot'] +_PHOTOMETRY_TIMING_STATS = { + stage: {'count': 0, 'total_s': 0.0} for stage in PHOTOMETRY_TIMING_STAGES +} + + +def reset_transform_timing_stats(): + for stage in TRANSFORM_TIMING_STAGES: + _TRANSFORM_TIMING_STATS[stage] = {'count': 0, 'success': 0, 'total_s': 0.0} + _TRANSFORM_TIMING_STATS['mask_loops_skipped'] = 0 + + +def reset_photometry_timing_stats(): + for stage in PHOTOMETRY_TIMING_STAGES: + _PHOTOMETRY_TIMING_STATS[stage] = {'count': 0, 'total_s': 0.0} + + +def _record_transform_stage_timing(stage, elapsed_s, success): + stage_stats = _TRANSFORM_TIMING_STATS[stage] + stage_stats['count'] += 1 + stage_stats['total_s'] += elapsed_s + if success: + stage_stats['success'] += 1 + + +def _record_photometry_stage_timing(stage, elapsed_s): + stage_stats = _PHOTOMETRY_TIMING_STATS[stage] + stage_stats['count'] += 1 + stage_stats['total_s'] += elapsed_s + + +def log_transform_timing_stats(prefix='Transformation timing summary'): + logged_any = False + lines = [] + + for stage in TRANSFORM_TIMING_STAGES: + stage_stats = _TRANSFORM_TIMING_STATS[stage] + if stage_stats['count'] == 0: + continue + + avg_ms = 1000.0 * stage_stats['total_s'] / stage_stats['count'] + lines.append( + f"{stage}: calls={stage_stats['count']}, success={stage_stats['success']}, " + f"avg_ms={avg_ms:.2f}, total_s={stage_stats['total_s']:.2f}" + ) + logged_any = True + + if _TRANSFORM_TIMING_STATS['mask_loops_skipped']: + lines.append(f"astroalign_mask_loops_skipped={_TRANSFORM_TIMING_STATS['mask_loops_skipped']}") + logged_any = True + + if logged_any: + log_info(f"{prefix}: " + " | ".join(lines)) + + +def log_photometry_timing_stats(prefix='Photometry timing summary'): + logged_any = False + lines = [] + + for stage in PHOTOMETRY_TIMING_STAGES: + stage_stats = _PHOTOMETRY_TIMING_STATS[stage] + if stage_stats['count'] == 0: + continue + + avg_ms = 1000.0 * stage_stats['total_s'] / stage_stats['count'] + lines.append( + f"{stage}: calls={stage_stats['count']}, avg_ms={avg_ms:.2f}, total_s={stage_stats['total_s']:.2f}" + ) + logged_any = True + + if logged_any: + log_info(f"{prefix}: " + " | ".join(lines)) + + +def log_reduction_timing_overview(prefix='Reduction timing overview'): + transform_total_s = sum(_TRANSFORM_TIMING_STATS[stage]['total_s'] for stage in TRANSFORM_TIMING_STAGES) + photometry_total_s = sum(_PHOTOMETRY_TIMING_STATS[stage]['total_s'] for stage in PHOTOMETRY_TIMING_STAGES) + combined_total_s = transform_total_s + photometry_total_s + if combined_total_s <= 0: + return + + dominant_bucket = 'transform' + if photometry_total_s > transform_total_s: + dominant_bucket = 'photometry' + + transform_pct = 100.0 * transform_total_s / combined_total_s + photometry_pct = 100.0 * photometry_total_s / combined_total_s + log_info( + f"{prefix}: transform_total_s={transform_total_s:.2f} ({transform_pct:.1f}%), " + f"photometry_total_s={photometry_total_s:.2f} ({photometry_pct:.1f}%), " + f"dominant={dominant_bucket}" + ) + + +def _display_filename(file_name): + return str(file_name).replace("\\", "/").rsplit("/", 1)[-1] + + +def format_plate_solution_reference(wcs_file): + return f"Here is the filename where we got the WCS from: {_display_filename(wcs_file)}" + + +# Aligns imaging data from .fits file to easily track the host and comparison star's positions +def transformation(image_data, file_name, roi=1, report_failure=True, reference_image=None): + start_time = perf_counter() + display_file_name = _display_filename(file_name) + + if report_failure: + plateStatus.setCurrentFilename(file_name) + + # crop image to ROI + if reference_image is None: + current_image = image_data[0] + reference_image = image_data[1] + else: + current_image = image_data + + reference_cache = _get_reference_transform_cache(reference_image, roi) + roix = reference_cache['roix'] + roiy = reference_cache['roiy'] + roi_reference = reference_cache['roi_reference'] + roi_current = current_image[roiy, roix] + + if roi_reference.shape != roi_current.shape or roi_reference.size == 0: + log.debug( + f"Warning: Following image failed pre-alignment checks in " + f"{perf_counter() - start_time:.2f}s - {display_file_name}" + ) + if report_failure: + plateStatus.alignmentError() + return SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) + + fft_tform = None + + # Fast FFT translation estimate before more expensive fallback stages. + # Most cadence images are dominated by small translations, so this stage + # can often solve alignment without invoking significantly slower + # feature-matching methods. + stage_start = perf_counter() + try: + shift, error, _ = phase_cross_correlation(roi_current, roi_reference, upsample_factor=4) + if np.all(np.isfinite(shift)) and np.isfinite(error): + max_shift = max(abs(shift[0]), abs(shift[1])) + if max_shift <= max(roi_current.shape): + fft_tform = SimilarityTransform(scale=1, rotation=0, translation=[-shift[1], -shift[0]]) + fft_high_confidence = error <= 0.1 and max_shift <= max(roi_current.shape) * 0.25 + _record_transform_stage_timing('fft_translation', perf_counter() - stage_start, True) + if fft_high_confidence: + log.debug( + f"Transformation solved via high-confidence FFT in " + f"{perf_counter() - start_time:.2f}s for {display_file_name}" + ) + return fft_tform + else: + _record_transform_stage_timing('fft_translation', perf_counter() - stage_start, False) + else: + _record_transform_stage_timing('fft_translation', perf_counter() - stage_start, False) + except Exception: + _record_transform_stage_timing('fft_translation', perf_counter() - stage_start, False) + + # Find transformation from .FITS files and catch exceptions if not able to. + stage_start = perf_counter() + try: + results = aa.find_transform(roi_reference, roi_current) + _record_transform_stage_timing('astroalign_direct', perf_counter() - stage_start, True) + log.debug( + f"Transformation solved via astroalign direct pass in " + f"{perf_counter() - start_time:.2f}s for {display_file_name}" + ) + return results[0] + except Exception: + _record_transform_stage_timing('astroalign_direct', perf_counter() - stage_start, False) + + # One cheap filtered pass to suppress noise and retry astroalign. + filtered_current = gaussian_filter(roi_current, sigma=1.0) + filtered_reference = reference_cache['filtered_reference'] + + stage_start = perf_counter() + try: + results = aa.find_transform(filtered_reference, filtered_current) + _record_transform_stage_timing('astroalign_filtered', perf_counter() - stage_start, True) + log.debug( + f"Transformation solved via filtered astroalign in " + f"{perf_counter() - start_time:.2f}s for {display_file_name}" + ) + return results[0] + except Exception: + _record_transform_stage_timing('astroalign_filtered', perf_counter() - stage_start, False) + + for p in [99, 98, 95, 90]: + base_mask1 = reference_cache['reference_masks'][p] + p_cur = np.percentile(roi_current, p) + base_mask0 = roi_current > p_cur + + for it in [2, 1, 0]: + # create binary mask to align image + mask1 = base_mask1 + mask0 = base_mask0 + + if it > 0: + mask1 = binary_erosion(mask1, iterations=it) + mask0 = binary_erosion(mask0, iterations=it) + + stage_start = perf_counter() + try: + results = aa.find_transform(mask1, mask0) + _record_transform_stage_timing('astroalign_mask', perf_counter() - stage_start, True) + log.debug( + f"Transformation solved via mask astroalign (p={p}, erode={it}) in " + f"{perf_counter() - start_time:.2f}s for {display_file_name}" + ) + return results[0] + except Exception: + _record_transform_stage_timing('astroalign_mask', perf_counter() - stage_start, False) + + stage_start = perf_counter() + try: + result1 = ird.similarity(roi_reference, roi_current, numiter=3) + _record_transform_stage_timing('imreg_dft', perf_counter() - stage_start, True) + log.debug( + f"Transformation solved via imreg_dft fallback in " + f"{perf_counter() - start_time:.2f}s for {display_file_name}" + ) + return SimilarityTransform(scale=result1['scale'], rotation=np.radians(result1['angle']), + translation=[-1 * result1['tvec'][1], -1 * result1['tvec'][0]]) + except Exception: + _record_transform_stage_timing('imreg_dft', perf_counter() - stage_start, False) + + if fft_tform is not None: + log.debug( + f"Transformation fell back to FFT translation in " + f"{perf_counter() - start_time:.2f}s for {display_file_name}" + ) + return fft_tform + + log.debug( + f"Warning: Following image failed to align in " + f"{perf_counter() - start_time:.2f}s - {display_file_name}" + ) + if report_failure: + plateStatus.alignmentError() + return SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) + + +def downsampled_fallback_transformation(image_data, file_name, report_failure=True, reference_image=None, + max_dimension=LEGACY_ALIGNMENT_MAX_DIMENSION): + """Run the legacy image transform on a reduced image and return full-resolution coordinates.""" + if reference_image is None: + current_image = np.asarray(image_data[0]) + reference_image = np.asarray(image_data[1]) + else: + current_image = np.asarray(image_data) + reference_image = np.asarray(reference_image) + + largest_dimension = max(current_image.shape[:2] + reference_image.shape[:2]) + max_dimension = max(1, int(max_dimension)) + downsample_factor = max(1, int(np.ceil(float(largest_dimension) / max_dimension))) + if downsample_factor == 1: + return transformation( + current_image, + file_name, + report_failure=report_failure, + reference_image=reference_image, + ) + + reduced_current = current_image[::downsample_factor, ::downsample_factor] + reduced_reference = reference_image[::downsample_factor, ::downsample_factor] + reduced_tform = transformation( + reduced_current, + file_name, + report_failure=report_failure, + reference_image=reduced_reference, + ) + return SimilarityTransform( + scale=float(reduced_tform.scale), + rotation=float(reduced_tform.rotation), + translation=np.asarray(reduced_tform.translation, dtype=float) * downsample_factor, + ) + +def load_image_data(file_name): + hdul = fits.open(name=file_name, memmap=False, cache=False, lazy_load_hdus=False, ignore_missing_end=True) + extension = 0 + image_header = hdul[extension].header + while image_header["NAXIS"] == 0: + extension += 1 + image_header = hdul[extension].header + + image_data = hdul[extension].data + hdul.close() + return image_data + + +def persistent_bad_pixel_count_threshold(frame_count, minimum_fraction=BAD_PIXEL_DETECTION_FRACTION): + if frame_count <= 0: + return 1 + return max(1, int(np.floor(float(minimum_fraction) * frame_count)) + 1) + + +def detect_frame_bad_pixels(image_data, + outlier_sigma=BAD_PIXEL_OUTLIER_SIGMA, + isolation_sigma=BAD_PIXEL_ISOLATION_SIGMA, + isolation_ratio=BAD_PIXEL_ISOLATION_RATIO): + values = np.asarray(image_data, dtype=float) + if values.ndim != 2 or values.size == 0: + return np.zeros(values.shape[:2], dtype=bool) + + finite_mask = np.isfinite(values) + if np.count_nonzero(finite_mask) < BAD_PIXEL_NEIGHBOR_FOOTPRINT.sum(): + return np.zeros(values.shape, dtype=bool) + + working = np.array(values, copy=True) + frame_median = bn.nanmedian(working[finite_mask]) + if not np.isfinite(frame_median): + frame_median = 0.0 + working[~finite_mask] = frame_median + + neighbor_median = median_filter(working, footprint=BAD_PIXEL_NEIGHBOR_FOOTPRINT, mode='mirror') + neighbor_max = maximum_filter(working, footprint=BAD_PIXEL_NEIGHBOR_FOOTPRINT, mode='mirror') + residual = working - neighbor_median + + global_scatter = robust_scatter(residual[finite_mask]) + if not np.isfinite(global_scatter) or global_scatter <= 0: + global_scatter = robust_scatter(working[finite_mask]) + if not np.isfinite(global_scatter) or global_scatter <= 0: + return np.zeros(values.shape, dtype=bool) + + local_scatter = 1.4826 * median_filter( + np.abs(residual), + footprint=BAD_PIXEL_NEIGHBOR_FOOTPRINT, + mode='mirror', + ) + diff_threshold = np.maximum(outlier_sigma * local_scatter, BAD_PIXEL_GLOBAL_SIGMA * global_scatter) + isolation_threshold = max(isolation_sigma * global_scatter, 1.0) + neighbor_scale = np.maximum(np.abs(neighbor_max), 1.0) + + with np.errstate(divide='ignore', invalid='ignore'): + isolation_ratio_values = np.divide(np.abs(working), neighbor_scale) + + return ( + finite_mask + & (residual > diff_threshold) + & ((working - neighbor_max) > isolation_threshold) + & (isolation_ratio_values >= isolation_ratio) + ) + + +_BAD_PIXEL_PRECHECK_POOL_CONTEXT = {} + + +def _bad_pixel_precheck_pool_initializer(generalDark, generalBias, generalFlat, + demosaic_fmt, demosaic_out, demosaic_mult): + global _BAD_PIXEL_PRECHECK_POOL_CONTEXT + suppress_inherited_tk_cleanup_in_worker() + _BAD_PIXEL_PRECHECK_POOL_CONTEXT = { + 'generalDark': generalDark, + 'generalBias': generalBias, + 'generalFlat': generalFlat, + 'demosaic_fmt': demosaic_fmt, + 'demosaic_out': demosaic_out, + 'demosaic_mult': demosaic_mult, + } + + +def _load_bad_pixel_precheck_worker_frame(file_name): + context = _BAD_PIXEL_PRECHECK_POOL_CONTEXT + hdul = fits.open(name=file_name, memmap=False, cache=False, lazy_load_hdus=False, ignore_missing_end=True) + extension = 0 + image_header = hdul[extension].header + while image_header["NAXIS"] == 0: + extension += 1 + image_header = hdul[extension].header + + image_data = hdul[extension].data + hdul.close() + + image_data = apply_cals( + image_data, + context.get('generalDark'), + context.get('generalBias'), + context.get('generalFlat'), + 1, + ) + image_data = demosaic_img( + image_data, + context.get('demosaic_fmt'), + context.get('demosaic_out'), + context.get('demosaic_mult'), + 1, + ) + return image_data + + +def _bad_pixel_precheck_task(task): + index, file_name = task + try: + frame_data = _load_bad_pixel_precheck_worker_frame(file_name) + except Exception as exc: + return { + 'index': index, + 'file_name': file_name, + 'usable': False, + 'error': str(exc), + } + + frame_mask = detect_frame_bad_pixels(frame_data) + if frame_mask.ndim != 2: + return { + 'index': index, + 'file_name': file_name, + 'usable': False, + 'not_2d': True, + } + + return { + 'index': index, + 'file_name': file_name, + 'usable': True, + 'mask': frame_mask, + } + + +def _merge_bad_pixel_precheck_mask(detection_counts, frame_mask, file_name): + frame_mask = np.asarray(frame_mask, dtype=bool) + if frame_mask.ndim != 2: + log_info( + f"Warning: skipping bad-pixel precheck for {_display_filename(file_name)} because the frame is not 2-D.", + warn=True, + ) + return detection_counts, False + + if detection_counts is None: + detection_counts = np.zeros(frame_mask.shape, dtype=np.uint32) + elif detection_counts.shape != frame_mask.shape: + log_info( + "Warning: skipping bad-pixel precheck for " + f"{_display_filename(file_name)} because its shape {frame_mask.shape} does not match " + f"the reference frame shape {detection_counts.shape}.", + warn=True, + ) + return detection_counts, False + + detection_counts += frame_mask.astype(np.uint32) + return detection_counts, True + + +def _scan_bad_pixel_precheck_frames_serial(inputfiles, frame_loader): + total_files = len(inputfiles) + detection_counts = None + scanned_files = 0 + + for index, file_name in enumerate(inputfiles): + plateStatus.setCurrentFilename(file_name) + try: + frame_data = frame_loader(file_name) + except Exception as exc: + log_info( + f"Warning: skipping bad-pixel precheck for {_display_filename(file_name)} ({exc}).", + warn=True, + ) + continue + + frame_mask = detect_frame_bad_pixels(frame_data) + detection_counts, usable = _merge_bad_pixel_precheck_mask(detection_counts, frame_mask, file_name) + if usable: + scanned_files += 1 + + completed = index + 1 + if completed == total_files or completed % BAD_PIXEL_PROGRESS_LOG_INTERVAL == 0: + log_info(f"Bad-pixel precheck progress: {completed}/{total_files}") + + return detection_counts, scanned_files + + +def _scan_bad_pixel_precheck_frames_multiprocess(inputfiles, max_processes, + generalDark=None, generalBias=None, generalFlat=None, + demosaic_fmt=None, demosaic_out=None, demosaic_mult=None): + total_files = len(inputfiles) + max_workers = min(max_processes, os.cpu_count() or 1, total_files, MAX_MULTIPROCESS_BAD_PIXEL_WORKERS) + detection_counts = None + scanned_files = 0 + + log_info( + "Using multiprocessing for bad-pixel precheck " + f"with {max_workers} worker(s) across {total_files} image(s)." + ) + + tasks = [(index, str(file_name)) for index, file_name in enumerate(inputfiles)] + with suppress_tk_cleanup_during_process_pool(): + with ProcessPoolExecutor( + max_workers=max_workers, + initializer=_bad_pixel_precheck_pool_initializer, + initargs=( + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + ), + ) as executor: + futures = [executor.submit(_bad_pixel_precheck_task, task) for task in tasks] + completed = 0 + for future in as_completed(futures): + result = future.result() + file_name = result.get('file_name') + if result.get('usable'): + detection_counts, usable = _merge_bad_pixel_precheck_mask( + detection_counts, + result.get('mask'), + file_name, + ) + if usable: + scanned_files += 1 + elif result.get('not_2d'): + log_info( + f"Warning: skipping bad-pixel precheck for {_display_filename(file_name)} " + "because the frame is not 2-D.", + warn=True, + ) + else: + log_info( + f"Warning: skipping bad-pixel precheck for {_display_filename(file_name)} " + f"({result.get('error')}).", + warn=True, + ) + + completed += 1 + if completed == total_files or completed % BAD_PIXEL_PROGRESS_LOG_INTERVAL == 0: + log_info(f"Bad-pixel precheck progress: {completed}/{total_files}") + + return detection_counts, scanned_files + + +def build_persistent_bad_pixel_map(inputfiles, frame_loader, save_directory=None, + minimum_fraction=BAD_PIXEL_DETECTION_FRACTION, + minimum_frames=BAD_PIXEL_PRECHECK_MIN_FRAMES, + max_processes=None, generalDark=None, generalBias=None, + generalFlat=None, demosaic_fmt=None, demosaic_out=None, + demosaic_mult=None): + inputfiles = list(inputfiles) + total_files = len(inputfiles) + if total_files < minimum_frames: + log_info( + f"Bad-pixel precheck skipped: only {total_files} frame(s); need at least {minimum_frames} frames.", + ) + return None + + try: + max_processes = int(max_processes) if max_processes is not None else None + except (TypeError, ValueError): + max_processes = None + + if max_processes is not None and max_processes > 1 and total_files > 1: + try: + detection_counts, scanned_files = _scan_bad_pixel_precheck_frames_multiprocess( + inputfiles, + max_processes, + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + demosaic_out=demosaic_out, + demosaic_mult=demosaic_mult, + ) + except Exception as exc: + log_info( + f"Warning: bad-pixel precheck multiprocessing failed ({exc}); falling back to serial scanning.", + warn=True, + ) + detection_counts, scanned_files = _scan_bad_pixel_precheck_frames_serial(inputfiles, frame_loader) + else: + detection_counts, scanned_files = _scan_bad_pixel_precheck_frames_serial(inputfiles, frame_loader) + + if detection_counts is None or scanned_files < minimum_frames: + log_info( + f"Bad-pixel precheck skipped: only {scanned_files} usable frame(s); need at least {minimum_frames}.", + warn=True, + ) + return None + + required_count = persistent_bad_pixel_count_threshold(scanned_files, minimum_fraction) + bad_pixel_mask = detection_counts >= required_count + coord_y, coord_x = np.nonzero(bad_pixel_mask) + + counts_path = None + mask_path = None + if save_directory is not None: + temp_dir = Path(save_directory) / "working_artifacts" + temp_dir.mkdir(parents=True, exist_ok=True) + counts_path = temp_dir / BAD_PIXEL_COUNTS_FILENAME + mask_path = temp_dir / BAD_PIXEL_MASK_FILENAME + fits.writeto(counts_path, detection_counts.astype(np.int32), overwrite=True) + fits.writeto(mask_path, bad_pixel_mask.astype(np.uint8), overwrite=True) + + threshold_percent = minimum_fraction * 100.0 + summary = ( + f"Bad-pixel precheck: identified {int(np.count_nonzero(bad_pixel_mask))} persistent bad pixel(s) " + f"after scanning {scanned_files}/{total_files} frame(s) with a >{threshold_percent:g}% recurrence threshold " + f"({required_count}+ detections)." + ) + if counts_path is not None and mask_path is not None: + summary += f" Saved {counts_path.name} and {mask_path.name} to working_artifacts/." + log_info(summary) + + return { + 'count_image': detection_counts, + 'mask': bad_pixel_mask, + 'coord_y': coord_y.astype(int), + 'coord_x': coord_x.astype(int), + 'required_count': required_count, + 'minimum_fraction': float(minimum_fraction), + 'frame_count': scanned_files, + 'counts_path': counts_path, + 'mask_path': mask_path, + } + + +def repair_bad_pixels_in_frame(image_data, bad_pixel_reference): + if bad_pixel_reference is None: + return image_data + + coord_y = bad_pixel_reference.get('coord_y') + coord_x = bad_pixel_reference.get('coord_x') + if coord_y is None or coord_x is None: + mask = np.asarray(bad_pixel_reference.get('mask'), dtype=bool) + if mask.size == 0: + return image_data + coord_y, coord_x = np.nonzero(mask) + + coord_y = np.asarray(coord_y, dtype=int) + coord_x = np.asarray(coord_x, dtype=int) + if coord_y.size == 0 or coord_x.size == 0: + return image_data + + repaired = np.array(image_data, dtype=float, copy=True) + valid_coords = ( + (coord_y >= 0) & (coord_y < repaired.shape[0]) + & (coord_x >= 0) & (coord_x < repaired.shape[1]) + ) + if not np.any(valid_coords): + return repaired + + coord_y = coord_y[valid_coords] + coord_x = coord_x[valid_coords] + repaired[coord_y, coord_x] = np.nan + + padded = np.pad(repaired, 1, mode='edge') + yp = coord_y + 1 + xp = coord_x + 1 + neighbors = np.stack([ + padded[yp - 1, xp - 1], + padded[yp - 1, xp], + padded[yp - 1, xp + 1], + padded[yp, xp - 1], + padded[yp, xp + 1], + padded[yp + 1, xp - 1], + padded[yp + 1, xp], + padded[yp + 1, xp + 1], + ], axis=0) + + fill_values = np.nanmedian(neighbors, axis=0) + if np.any(~np.isfinite(fill_values)): + frame_median = bn.nanmedian(repaired) + if not np.isfinite(frame_median): + frame_median = 0.0 + fill_values[~np.isfinite(fill_values)] = frame_median + + repaired[coord_y, coord_x] = fill_values + return repaired + + +def transformation_task(i, file_name, reference_file): + if i == 0: + return i, SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) + + image_data = load_image_data(file_name) + reference_image = load_image_data(reference_file) + # Multiprocess pre-computation should not emit plate-status warnings; the + # serial reduction path decides whether the fallback transform is needed. + return i, downsampled_fallback_transformation( + image_data, + file_name, + report_failure=False, + reference_image=reference_image, + ) + + +_TRANSFORM_REFERENCE_IMAGE = None +_TRANSFORM_REFERENCE_CACHE = None + + +def _build_reference_transform_cache(reference_image, roi): + height = reference_image.shape[0] + width = reference_image.shape[1] + roix = slice(int(width * (0.5 - roi / 2)), int(width * (0.5 + roi / 2))) + roiy = slice(int(height * (0.5 - roi / 2)), int(height * (0.5 + roi / 2))) + + roi_reference = reference_image[roiy, roix] + + cache = { + 'ref_id': id(reference_image), + 'shape': reference_image.shape, + 'roi': roi, + 'roix': roix, + 'roiy': roiy, + 'roi_reference': roi_reference, + 'filtered_reference': gaussian_filter(roi_reference, sigma=1.0), + } + + reference_masks = {} + for p in [99, 98, 95, 90]: + p_ref = np.percentile(roi_reference, p) + reference_masks[p] = roi_reference > p_ref + cache['reference_masks'] = reference_masks + + return cache + + +def _get_reference_transform_cache(reference_image, roi): + global _TRANSFORM_REFERENCE_CACHE + + if (_TRANSFORM_REFERENCE_CACHE is None + or _TRANSFORM_REFERENCE_CACHE['ref_id'] != id(reference_image) + or _TRANSFORM_REFERENCE_CACHE['shape'] != reference_image.shape + or _TRANSFORM_REFERENCE_CACHE['roi'] != roi): + _TRANSFORM_REFERENCE_CACHE = _build_reference_transform_cache(reference_image, roi) + + return _TRANSFORM_REFERENCE_CACHE + + +def _transformation_pool_initializer(reference_file): + global _TRANSFORM_REFERENCE_IMAGE, _TRANSFORM_REFERENCE_CACHE + suppress_inherited_tk_cleanup_in_worker() + _TRANSFORM_REFERENCE_IMAGE = load_image_data(reference_file) + _TRANSFORM_REFERENCE_CACHE = None + + +def transformation_task_with_cached_reference(i, file_name): + image_data = load_image_data(file_name) + return i, downsampled_fallback_transformation( + image_data, + file_name, + report_failure=False, + reference_image=_TRANSFORM_REFERENCE_IMAGE, + ) + + +class _ParallelPlateStatusRecorder: + def __init__(self): + self.warnings = [] + + def setCurrentFilename(self, filename): + return self + + def outOfFrameWarning(self, starIndex): + self.warnings.append(('out_of_frame', int(starIndex), np.nan, np.nan)) + + def lowFluxAmplitudeWarning(self, starIndex, xc, yc): + self.warnings.append(('low_flux', int(starIndex), float(xc), float(yc))) + + def alignmentError(self): + self.warnings.append(('alignment_error', -1, np.nan, np.nan)) + + +_PLATE_STATUS_SWAP_LOCK = threading.RLock() +_ALIGNMENT_POOL_CONTEXT = {} + + +def _alignment_pool_initializer(reference_file, generalDark, generalBias, generalFlat, + demosaic_fmt, demosaic_out, demosaic_mult, bad_pixel_reference): + global _ALIGNMENT_POOL_CONTEXT, _TRANSFORM_REFERENCE_IMAGE, _TRANSFORM_REFERENCE_CACHE + suppress_inherited_tk_cleanup_in_worker() + _ALIGNMENT_POOL_CONTEXT = { + 'generalDark': generalDark, + 'generalBias': generalBias, + 'generalFlat': generalFlat, + 'demosaic_fmt': demosaic_fmt, + 'demosaic_out': demosaic_out, + 'demosaic_mult': demosaic_mult, + 'bad_pixel_reference': bad_pixel_reference, + 'reference_file': str(reference_file), + } + # WCS-first jobs do not need the large reference image. Load it lazily only + # inside a worker that is actually assigned a legacy alignment fallback. + _TRANSFORM_REFERENCE_IMAGE = None + _TRANSFORM_REFERENCE_CACHE = None + + +def _alignment_worker_reference_image(): + global _TRANSFORM_REFERENCE_IMAGE + if _TRANSFORM_REFERENCE_IMAGE is None: + context = _ALIGNMENT_POOL_CONTEXT + _TRANSFORM_REFERENCE_IMAGE = load_calibrated_reduction_image( + context['reference_file'], + context.get('generalDark'), + context.get('generalBias'), + context.get('generalFlat'), + context.get('demosaic_fmt'), + context.get('demosaic_out'), + context.get('demosaic_mult'), + bad_pixel_reference=context.get('bad_pixel_reference'), + ) + return _TRANSFORM_REFERENCE_IMAGE + + +def _load_alignment_worker_frame(file_name): + context = _ALIGNMENT_POOL_CONTEXT + use_memmap = can_memmap_aperture_tuning_cutouts( + generalDark=context.get('generalDark'), + generalBias=context.get('generalBias'), + generalFlat=context.get('generalFlat'), + demosaic_fmt=context.get('demosaic_fmt'), + bad_pixel_reference=context.get('bad_pixel_reference'), + ) + hdul = fits.open( + name=file_name, + memmap=use_memmap, + cache=False, + lazy_load_hdus=use_memmap, + ignore_missing_end=True, + ) + extension = 0 + image_header = hdul[extension].header + while image_header["NAXIS"] == 0: + extension += 1 + image_header = hdul[extension].header + + if use_memmap and not fits_header_supports_memmap(image_header): + hdul.close() + hdul = fits.open( + name=file_name, + memmap=False, + cache=False, + lazy_load_hdus=False, + ignore_missing_end=True, + ) + extension = 0 + image_header = hdul[extension].header + while image_header["NAXIS"] == 0: + extension += 1 + image_header = hdul[extension].header + use_memmap = False + image_data = hdul[extension].data + hdul.close() + + if not use_memmap: + image_data = apply_cals( + image_data, + context.get('generalDark'), + context.get('generalBias'), + context.get('generalFlat'), + 1, + ) + image_data = demosaic_img( + image_data, + context.get('demosaic_fmt'), + context.get('demosaic_out'), + context.get('demosaic_mult'), + 1, + ) + image_data = repair_bad_pixels_in_frame(image_data, context.get('bad_pixel_reference')) + return image_header, image_data + + +def _pointing_precheck_alignment_task(task): + i, file_name, reference_anchor = task + try: + _, image_data = _load_alignment_worker_frame(file_name) + if getattr(image_data, "ndim", 0) != 2: + return { + 'index': i, + 'file_name': file_name, + 'usable': False, + 'position': np.array([np.nan, np.nan], dtype=float), + 'transform': None, + } + + tform = downsampled_fallback_transformation( + image_data, + file_name, + report_failure=False, + reference_image=_alignment_worker_reference_image(), + ) + mapped_anchor = np.asarray(tform(reference_anchor), dtype=float).reshape(-1, 2)[0] + usable = bool(np.all(np.isfinite(mapped_anchor))) + return { + 'index': i, + 'file_name': file_name, + 'usable': usable, + 'position': mapped_anchor, + 'transform': tform if usable else None, + } + except Exception as exc: + return { + 'index': i, + 'file_name': file_name, + 'usable': False, + 'position': np.array([np.nan, np.nan], dtype=float), + 'transform': None, + 'error': str(exc), + } + + +def build_multiprocess_pointing_precheck_transforms(inputfiles, max_processes, reference_anchor, + generalDark=None, generalBias=None, generalFlat=None, + demosaic_fmt=None, demosaic_out=None, demosaic_mult=None): + total_jobs = len(inputfiles) + positions = np.full((total_jobs, 2), np.nan, dtype=float) + usable_mask = np.zeros(total_jobs, dtype=bool) + alignment_transforms = {} + if total_jobs == 0: + return positions, usable_mask, alignment_transforms + + reference_anchor = np.asarray(reference_anchor, dtype=float).reshape(-1, 2) + positions[0] = reference_anchor[0] + usable_mask[0] = True + alignment_transforms[str(inputfiles[0])] = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) + if total_jobs == 1: + return positions, usable_mask, alignment_transforms + + max_workers = min(max_processes, os.cpu_count() or 1, total_jobs - 1, MAX_MULTIPROCESS_TRANSFORM_WORKERS) + log_info( + "Using multiprocessing for pointing precheck alignment " + f"with {max_workers} worker(s) across {total_jobs} image(s)." + ) + + tasks = [ + (i, str(file_name), reference_anchor) + for i, file_name in enumerate(inputfiles) + if i != 0 + ] + + with suppress_tk_cleanup_during_process_pool(): + with ProcessPoolExecutor( + max_workers=max_workers, + initializer=_alignment_pool_initializer, + initargs=( + str(inputfiles[0]), + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + None, + ), + ) as executor: + futures = [executor.submit(_pointing_precheck_alignment_task, task) for task in tasks] + completed = 1 + for future in as_completed(futures): + result = future.result() + index = result['index'] + if result.get('usable'): + positions[index] = result['position'] + usable_mask[index] = True + alignment_transforms[result['file_name']] = result['transform'] + completed += 1 + if completed == total_jobs or completed % 10 == 0: + log_info(f"Pointing precheck alignment progress: {completed}/{total_jobs}") + + return positions, usable_mask, alignment_transforms + + +def _fit_alignment_candidate_psfs(image_data, predicted_coords, target_fast_centroid, frame_fast_centroid, + previous_psf_rows=None): + global plateStatus + predicted_coords = np.asarray(predicted_coords, dtype=float) + previous_psf_rows = {} if previous_psf_rows is None else dict(previous_psf_rows) + with _PLATE_STATUS_SWAP_LOCK: + original_plate_status = plateStatus + recorder = _ParallelPlateStatusRecorder() + plateStatus = recorder + try: + psf_rows = { + 'target': fit_centroid_or_warn_out_of_frame( + image_data, + choose_centroid_seed_position(predicted_coords[0], previous_psf_rows.get('target')), + 0, + fast_mode=target_fast_centroid, + ) + } + for comp_idx in range(max(0, predicted_coords.shape[0] - 1)): + comp_key = f"comp{comp_idx + 1}" + psf_rows[f"comp{comp_idx + 1}"] = fit_centroid_or_warn_out_of_frame( + image_data, + choose_centroid_seed_position(predicted_coords[comp_idx + 1], previous_psf_rows.get(comp_key)), + comp_idx + 1, + fast_mode=frame_fast_centroid, + ) + return { + 'coords': predicted_coords, + 'psf_rows': psf_rows, + 'warnings': list(recorder.warnings), + } + finally: + plateStatus = original_plate_status + + +def _parallel_alignment_task(task): + ( + i, + file_name, + target_and_comp_pixels, + target_and_comp_radec, + ignore_header_wcs, + target_fast_centroid, + frame_fast_centroid, + compute_fallback_transform, + first_frame_uses_input_comp_pixels, + precomputed_fallback_transform, + ) = task + + target_and_comp_pixels = np.asarray(target_and_comp_pixels, dtype=float) + if target_and_comp_radec is not None: + target_and_comp_radec = np.asarray(target_and_comp_radec, dtype=float) + + image_header, image_data = _load_alignment_worker_frame(file_name) + result = { + 'index': i, + 'file_name': file_name, + 'wcs': None, + 'fallback': None, + } + + if not ignore_header_wcs and target_and_comp_radec is not None: + try: + wcs_hdr = search_wcs_from_header(image_header) + if wcs_hdr.is_celestial: + pix_x, pix_y = wcs_hdr.world_to_pixel_values( + target_and_comp_radec[:, 0], + target_and_comp_radec[:, 1], + ) + pix_x = np.asarray(pix_x, dtype=float).reshape(-1) + pix_y = np.asarray(pix_y, dtype=float).reshape(-1) + projected_coords = np.column_stack((pix_x, pix_y)) + + wcs_candidate = _fit_alignment_candidate_psfs( + image_data, + projected_coords, + target_fast_centroid, + frame_fast_centroid, + ) + wcs_candidate['projected_off_frame'] = any_projected_coord_out_of_frame( + projected_coords, + image_data.shape, + ) + result['wcs'] = wcs_candidate + except Exception as exc: + result['wcs_error'] = str(exc) + + if precomputed_fallback_transform is not None or compute_fallback_transform: + if precomputed_fallback_transform is not None: + tform = precomputed_fallback_transform + elif i == 0: + tform = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) + else: + tform = downsampled_fallback_transformation( + image_data, + file_name, + report_failure=False, + reference_image=_alignment_worker_reference_image(), + ) + transformed_coords = np.asarray(tform(target_and_comp_pixels), dtype=float) + result['fallback'] = _fit_alignment_candidate_psfs( + image_data, + transformed_coords, + target_fast_centroid, + frame_fast_centroid, + ) + + return result + + +def _replay_parallel_alignment_warnings(file_name, warnings): + if not warnings: + return + + plateStatus.setCurrentFilename(file_name) + for warning_type, star_index, xc, yc in warnings: + if warning_type == 'out_of_frame': + plateStatus.outOfFrameWarning(star_index) + elif warning_type == 'low_flux': + plateStatus.lowFluxAmplitudeWarning(star_index, xc, yc) + elif warning_type == 'alignment_error': + plateStatus.alignmentError() + + +def _store_alignment_candidate_psfs(candidate, frame_index, psf_data, comp_keys): + psf_data['target'][frame_index] = candidate['psf_rows']['target'] + for comp_idx, comp_key in enumerate(comp_keys): + psf_data[comp_key][frame_index] = candidate['psf_rows'].get( + f"comp{comp_idx + 1}", + _nan_psf_result(), + ) + + +def _update_reference_comp_offsets(psf_data, tar_comp_dist, comp_keys): + target_row = psf_data['target'][0] + if not centroid_position_is_finite(target_row): + return + + for comp_key in comp_keys: + comp_row = psf_data[comp_key][0] + if not centroid_position_is_finite(comp_row): + continue + tar_comp_dist[comp_key][0] = abs(int(comp_row[0]) - int(target_row[0])) + tar_comp_dist[comp_key][1] = abs(int(comp_row[1]) - int(target_row[1])) + + +def _candidate_seed_position(candidate, star_index): + if candidate is None: + return None + + try: + coords = np.asarray(candidate.get('coords'), dtype=float) + except (TypeError, ValueError, AttributeError): + return None + + if coords.ndim == 1: + if coords.size % 2 != 0: + return None + coords = coords.reshape(-1, 2) + if coords.ndim != 2 or coords.shape[1] < 2 or star_index >= coords.shape[0]: + return None + + seed = coords[star_index, :2] + if np.all(np.isfinite(seed)): + return seed + return None + + +def alignment_candidate_quality_score(candidate, comp_keys=None, previous_target_psf_row=None, + previous_comp_psf_rows=None, expected_offsets=None, + width_max_comp_ratio=PSF_ALIGNMENT_TARGET_WIDTH_MAX_COMP_RATIO): + if candidate is None: + return np.inf + + psf_rows = candidate.get('psf_rows') if isinstance(candidate, dict) else None + if not isinstance(psf_rows, dict): + return np.inf + + comp_keys = [] if comp_keys is None else list(comp_keys) + previous_comp_psf_rows = {} if previous_comp_psf_rows is None else dict(previous_comp_psf_rows) + expected_offsets = {} if expected_offsets is None else dict(expected_offsets) + + target_row = psf_rows.get('target', _nan_psf_result()) + target_score = psf_solution_quality_score( + target_row, + seed_pos=_candidate_seed_position(candidate, 0), + ) + if not np.isfinite(target_score): + return np.inf + + score = float(target_score) + target_sigma = psf_sigma_from_fit(target_row) + comp_sigmas = [] + geometry_test_count = 0 + geometry_match_count = 0 + + for comp_idx, comp_key in enumerate(comp_keys): + row = psf_rows.get(f"comp{comp_idx + 1}", _nan_psf_result()) + comp_score = psf_solution_quality_score( + row, + seed_pos=_candidate_seed_position(candidate, comp_idx + 1), + ) + if np.isfinite(comp_score): + score += 0.25 * float(comp_score) + comp_sigma = psf_sigma_from_fit(row) + if np.isfinite(comp_sigma) and comp_sigma > 0: + comp_sigmas.append(comp_sigma) + elif comp_keys: + score += 0.75 + + expected_offset = expected_offsets.get(comp_key) + if expected_offset is None: + continue + + try: + expected_dx = float(expected_offset[0]) + expected_dy = float(expected_offset[1]) + except (TypeError, ValueError, IndexError): + continue + if expected_dx == 0.0 and expected_dy == 0.0: + continue + + geometry_test_count += 1 + if centroid_offset_matches_reference(row, target_row, expected_dx, expected_dy): + geometry_match_count += 1 + + previous_comp_row = previous_comp_psf_rows.get(comp_key) + if centroid_position_is_finite(row) and centroid_position_is_finite(previous_comp_row): + comp_jump = float(np.hypot(float(row[0]) - float(previous_comp_row[0]), + float(row[1]) - float(previous_comp_row[1]))) + score += min(comp_jump / 20.0, 1.0) + + if geometry_test_count: + geometry_miss_fraction = (geometry_test_count - geometry_match_count) / geometry_test_count + score += 2.0 * geometry_miss_fraction + + if np.isfinite(target_sigma) and target_sigma > 0 and comp_sigmas: + comp_sigma_center = float(bn.nanmedian(np.asarray(comp_sigmas, dtype=float))) + if np.isfinite(comp_sigma_center) and comp_sigma_center > 0: + width_ratio = target_sigma / comp_sigma_center + if ( + not np.isfinite(width_ratio) + or width_ratio > float(width_max_comp_ratio) + ): + return np.inf + score += 0.5 * abs(np.log(width_ratio)) + + if centroid_position_is_finite(target_row) and centroid_position_is_finite(previous_target_psf_row): + target_jump = float(np.hypot(float(target_row[0]) - float(previous_target_psf_row[0]), + float(target_row[1]) - float(previous_target_psf_row[1]))) + score += min(target_jump / 20.0, 1.0) + + return float(score) + + +def select_alignment_candidate(result, frame_index, psf_data, tar_comp_dist, comp_keys): + wcs_candidate = result.get('wcs') if isinstance(result, dict) else None + fallback_candidate = result.get('fallback') if isinstance(result, dict) else None + selected_candidate = None + selected_source = 'fallback' + + previous_target_psf_row = None if frame_index == 0 else psf_data['target'][frame_index - 1] + previous_comp_psf_rows = {} + if frame_index != 0: + previous_comp_psf_rows = {comp_key: psf_data[comp_key][frame_index - 1] for comp_key in comp_keys} + + wcs_alignment_decision = {'use_wcs_alignment': False, 'reason': 'no_wcs_candidate'} + if wcs_candidate is not None: + comp_psf_rows = { + comp_key: wcs_candidate['psf_rows'].get(f"comp{comp_idx + 1}", _nan_psf_result()) + for comp_idx, comp_key in enumerate(comp_keys) + } + + wcs_alignment_decision = should_keep_header_wcs_alignment( + wcs_candidate.get('projected_off_frame', False), + frame_index, + wcs_candidate['psf_rows']['target'], + previous_target_psf_row=previous_target_psf_row, + comp_psf_rows=comp_psf_rows, + previous_comp_psf_rows=previous_comp_psf_rows, + expected_offsets=tar_comp_dist, + ) + + wcs_score = alignment_candidate_quality_score( + wcs_candidate, + comp_keys=comp_keys, + previous_target_psf_row=previous_target_psf_row, + previous_comp_psf_rows=previous_comp_psf_rows, + expected_offsets=tar_comp_dist, + ) + fallback_score = alignment_candidate_quality_score( + fallback_candidate, + comp_keys=comp_keys, + previous_target_psf_row=previous_target_psf_row, + previous_comp_psf_rows=previous_comp_psf_rows, + expected_offsets=tar_comp_dist, + ) + + if ( + wcs_candidate is not None + and wcs_alignment_decision.get('use_wcs_alignment') + and np.isfinite(wcs_score) + and ( + not np.isfinite(fallback_score) + or wcs_score <= fallback_score + PSF_ALIGNMENT_CANDIDATE_SELECTION_MARGIN + ) + ): + selected_candidate = wcs_candidate + selected_source = 'wcs' + elif fallback_candidate is not None and np.isfinite(fallback_score): + selected_candidate = fallback_candidate + selected_source = 'fallback' + elif wcs_candidate is not None and np.isfinite(wcs_score): + selected_candidate = wcs_candidate + selected_source = 'wcs' + elif fallback_candidate is not None: + selected_candidate = fallback_candidate + selected_source = 'fallback' + elif wcs_candidate is not None: + selected_candidate = wcs_candidate + selected_source = 'wcs' + + if selected_candidate is None: + selected_candidate = { + 'psf_rows': {'target': _nan_psf_result()}, + 'warnings': [('alignment_error', -1, np.nan, np.nan)], + } + + return selected_source, selected_candidate, { + 'wcs_score': wcs_score, + 'fallback_score': fallback_score, + 'wcs_decision': wcs_alignment_decision, + } + + +def apply_parallel_alignment_result(result, frame_index, psf_data, tar_comp_dist, comp_keys): + selected_source, selected_candidate, _ = select_alignment_candidate( + result, + frame_index, + psf_data, + tar_comp_dist, + comp_keys, + ) + + _store_alignment_candidate_psfs(selected_candidate, frame_index, psf_data, comp_keys) + _replay_parallel_alignment_warnings(result.get('file_name'), selected_candidate.get('warnings')) + if frame_index == 0: + _update_reference_comp_offsets(psf_data, tar_comp_dist, comp_keys) + + return selected_source + + +def wcs_alignment_candidate_is_acceptable(result, frame_index, psf_data, tar_comp_dist, comp_keys): + if not isinstance(result, dict) or result.get('wcs') is None: + return False + + _, _, diagnostics = select_alignment_candidate( + result, + frame_index, + psf_data, + tar_comp_dist, + comp_keys, + ) + return bool( + diagnostics['wcs_decision'].get('use_wcs_alignment') + and np.isfinite(diagnostics['wcs_score']) + ) + + +def log_wcs_authoritative_candidate_diagnostics(result, frame_index, psf_data, tar_comp_dist, comp_keys): + file_name = _display_filename(result.get('file_name')) if isinstance(result, dict) else '' + if not isinstance(result, dict) or result.get('wcs') is None: + detail = result.get('wcs_error') if isinstance(result, dict) else None + suffix = f" ({detail})" if detail else "" + log.debug( + f"WCS-authoritative frame has no usable WCS candidate for {file_name}{suffix}; " + "pixel alignment fallback is disabled." + ) + return + + _, _, diagnostics = select_alignment_candidate( + result, + frame_index, + psf_data, + tar_comp_dist, + comp_keys, + ) + decision = diagnostics.get('wcs_decision', {}) + log.debug( + "WCS-authoritative mode retained the frame-WCS-derived candidate without pixel alignment " + f"for {file_name}: reason={decision.get('reason', 'unknown')}, " + f"geometry={decision.get('geometry_match_count', 0)}/" + f"{decision.get('geometry_test_count', 0)}, wcs_score={diagnostics.get('wcs_score')}." + ) + + +def classify_wcs_fallback_frames(results, target_and_comp_pixels): + """Return missing and rejected WCS frame indices without running legacy alignment.""" + target_and_comp_pixels = np.asarray(target_and_comp_pixels, dtype=float).reshape(-1, 2) + comp_keys = [f"comp{comp_idx + 1}" for comp_idx in range(max(0, len(target_and_comp_pixels) - 1))] + psf_data = {'target': np.zeros((len(results), 7), dtype=float)} + tar_comp_dist = {} + for comp_idx, comp_key in enumerate(comp_keys): + psf_data[comp_key] = np.zeros((len(results), 7), dtype=float) + tar_comp_dist[comp_key] = np.abs( + target_and_comp_pixels[comp_idx + 1] - target_and_comp_pixels[0] + ) + + missing_wcs_indices = [] + rejected_wcs_indices = [] + for frame_index, result in enumerate(results): + if result is None or result.get('wcs') is None: + missing_wcs_indices.append(frame_index) + elif wcs_alignment_candidate_is_acceptable( + result, + frame_index, + psf_data, + tar_comp_dist, + comp_keys, + ): + _store_alignment_candidate_psfs(result['wcs'], frame_index, psf_data, comp_keys) + if frame_index == 0: + _update_reference_comp_offsets(psf_data, tar_comp_dist, comp_keys) + continue + else: + rejected_wcs_indices.append(frame_index) + + # Keep the last accepted solution as the continuity reference while the + # rejected frame waits for its fallback result. + if frame_index > 0: + psf_data['target'][frame_index] = psf_data['target'][frame_index - 1] + for comp_key in comp_keys: + psf_data[comp_key][frame_index] = psf_data[comp_key][frame_index - 1] + + return missing_wcs_indices, rejected_wcs_indices + + +def _run_multiprocess_alignment_task_batch(tasks, max_processes, reference_file, + generalDark=None, generalBias=None, generalFlat=None, + demosaic_fmt=None, demosaic_out=None, demosaic_mult=None, + bad_pixel_reference=None, progress_label='alignment'): + total_jobs = len(tasks) + if total_jobs == 0: + return {} + + batch_start = perf_counter() + max_workers = min(max_processes, os.cpu_count() or 1, total_jobs, MAX_MULTIPROCESS_TRANSFORM_WORKERS) + log_info( + f"Using multiprocessing for {progress_label} " + f"with {max_workers} worker(s) across {total_jobs} image(s)." + ) + + results = {} + with suppress_tk_cleanup_during_process_pool(): + with ImageProcessPoolExecutor( + max_workers=max_workers, + initializer=_alignment_pool_initializer, + initargs=( + str(reference_file), + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + bad_pixel_reference, + ), + ) as executor: + futures = [executor.submit(_parallel_alignment_task, task) for task in tasks] + completed = 0 + for future in as_completed(futures): + result = future.result() + results[result['index']] = result + completed += 1 + if completed == total_jobs or completed % 10 == 0: + log_info(f"Multiprocessing {progress_label} progress: {completed}/{total_jobs}") + + elapsed_seconds = perf_counter() - batch_start + log_info( + f"Multiprocessing {progress_label} completed {total_jobs} image(s) in " + f"{elapsed_seconds:.2f}s ({1000.0 * elapsed_seconds / total_jobs:.1f} ms/image wall time)." + ) + return results + + +def build_multiprocess_alignment_results(inputfiles, max_processes, target_and_comp_pixels, + target_and_comp_radec=None, ignore_header_wcs=False, + generalDark=None, generalBias=None, generalFlat=None, + demosaic_fmt=None, demosaic_out=None, demosaic_mult=None, + bad_pixel_reference=None, use_fast_centroid_cadence=False, + use_adaptive_apertures=False, compute_fallback_transform=False, + first_frame_uses_input_comp_pixels=False, + precomputed_fallback_transforms=None): + total_jobs = len(inputfiles) + if total_jobs == 0: + return [] + + wcs_tasks = [] + for i, file_name in enumerate(inputfiles): + frame_fast_centroid = should_use_fast_centroid(i) if use_fast_centroid_cadence else False + target_fast_centroid = ( + should_use_fast_target_centroid(i, adaptive_apertures=use_adaptive_apertures) + if use_fast_centroid_cadence else False + ) + wcs_tasks.append(( + i, + str(file_name), + target_and_comp_pixels, + target_and_comp_radec, + ignore_header_wcs, + target_fast_centroid, + frame_fast_centroid, + False, + first_frame_uses_input_comp_pixels, + None, + )) + + wcs_results_by_index = _run_multiprocess_alignment_task_batch( + wcs_tasks, + max_processes, + inputfiles[0], + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + demosaic_out=demosaic_out, + demosaic_mult=demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + progress_label='WCS coordinate projection', + ) + results = [wcs_results_by_index.get(i) for i in range(total_jobs)] + if not compute_fallback_transform: + return results + + missing_wcs_indices, rejected_wcs_indices = classify_wcs_fallback_frames( + results, + target_and_comp_pixels, + ) + fallback_indices = sorted(missing_wcs_indices + rejected_wcs_indices) + log_info( + f"WCS-first alignment accepted {total_jobs - len(fallback_indices)}/{total_jobs} frame(s); " + f"queued {len(fallback_indices)} legacy fallback(s) " + f"(missing WCS: {len(missing_wcs_indices)}, rejected WCS: {len(rejected_wcs_indices)}). " + f"Legacy fallback images are downsampled to at most {LEGACY_ALIGNMENT_MAX_DIMENSION} pixels " + "on their longest side." + ) + if not fallback_indices: + return results + + fallback_tasks = [] + for i in fallback_indices: + file_name = inputfiles[i] + precomputed_fallback_transform = None + if precomputed_fallback_transforms: + precomputed_fallback_transform = precomputed_fallback_transforms.get(str(file_name)) + frame_fast_centroid = should_use_fast_centroid(i) if use_fast_centroid_cadence else False + target_fast_centroid = ( + should_use_fast_target_centroid(i, adaptive_apertures=use_adaptive_apertures) + if use_fast_centroid_cadence else False + ) + fallback_tasks.append(( + i, + str(file_name), + target_and_comp_pixels, + target_and_comp_radec, + True, + target_fast_centroid, + frame_fast_centroid, + True, + first_frame_uses_input_comp_pixels, + precomputed_fallback_transform, + )) + + fallback_results = _run_multiprocess_alignment_task_batch( + fallback_tasks, + max_processes, + inputfiles[0], + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + demosaic_out=demosaic_out, + demosaic_mult=demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + progress_label='legacy alignment fallback', + ) + for i in fallback_indices: + fallback_result = fallback_results.get(i) + if fallback_result is not None: + results[i]['fallback'] = fallback_result.get('fallback') + + return results + + +MAX_MULTIPROCESS_TRANSFORM_WORKERS = 8 +SPARSE_MISSING_WCS_DROP_THRESHOLD = 0.03 +POINTING_REJECTION_MIN_FRAMES = 5 +POINTING_REJECTION_MAX_ITERS = 5 + +# Automatic aperture-grid tuning constants (in PSF sigma units) +GAUSSIAN_SIGMA_TO_FWHM = 2.355 +APERTURE_MIN_FWHM_MULTIPLIER = 0.5 +APERTURE_MAX_FWHM_MULTIPLIER = 2.0 +APERTURE_SIGMA_MIN = APERTURE_MIN_FWHM_MULTIPLIER * GAUSSIAN_SIGMA_TO_FWHM +APERTURE_SIGMA_MAX = APERTURE_MAX_FWHM_MULTIPLIER * GAUSSIAN_SIGMA_TO_FWHM +ANNULUS_SIGMA_MIN = 6.0 +ANNULUS_SIGMA_MAX = 15.0 +SKY_ANNULUS_MIN_GAP_PIXELS = 2.0 +SKY_ANNULUS_MIN_FWHM_MULTIPLIER = 3.0 +SKY_ANNULUS_MIN_EFFECTIVE_PIXELS = 250.0 +SKY_BACKGROUND_SIGMA_CLIP = 3.0 +SKY_BACKGROUND_SIGMA_CLIP_MAX_ITERS = 3 +APERTURE_CORRECTION_DETECTION_SIGMA = 5.0 +APERTURE_CORRECTION_MAX_DETECTED_STARS = 500 +APERTURE_CORRECTION_PEAK_TEST_LIMIT = APERTURE_CORRECTION_MAX_DETECTED_STARS * 50 +APERTURE_CORRECTION_PEAK_BLOCK_SIZE = 512 +APERTURE_CORRECTION_PEAK_BLOCK_LIMIT = 128 +APERTURE_CORRECTION_MAX_STARS = 60 +APERTURE_CORRECTION_MIN_STARS = 3 +APERTURE_CORRECTION_MIN_SEPARATION_FWHM = 5.0 +APERTURE_CORRECTION_MIN_BORDER_PIXELS = 20.0 +APERTURE_CORRECTION_MAX_FACTOR = 10.0 +APERTURE_AUTOTUNE_COARSE_APER_POINTS = 5 +APERTURE_AUTOTUNE_COARSE_ANNULUS_POINTS = 4 +APERTURE_AUTOTUNE_REFINED_APER_POINTS = 6 +APERTURE_AUTOTUNE_REFINED_ANNULUS_POINTS = 6 +APERTURE_AUTOTUNE_APER_HALF_WIDTH_SIGMA = 0.9 +APERTURE_AUTOTUNE_ANNULUS_HALF_WIDTH_SIGMA = 2.0 +APERTURE_AUTOTUNE_MIN_FRAMES = 8 +APERTURE_AUTOTUNE_MAX_FRAMES = 24 + +# Refit full PSF moments periodically; use a faster moment estimator for most frames. +CENTROID_FULL_FIT_CADENCE = 6 + + +def build_multiprocess_transformations(inputfiles, max_processes): + reference_file = str(inputfiles[0]) + max_workers = min(max_processes, os.cpu_count() or 1, len(inputfiles), MAX_MULTIPROCESS_TRANSFORM_WORKERS) + transforms = {} + total_jobs = len(inputfiles) + + log_info( + "Using multiprocessing for transformations " + f"with {max_workers} worker(s) across {total_jobs} image(s)." + ) + + transforms[0] = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) + + with suppress_tk_cleanup_during_process_pool(): + with ProcessPoolExecutor(max_workers=max_workers, initializer=_transformation_pool_initializer, + initargs=(reference_file,)) as executor: + futures = [executor.submit(transformation_task_with_cached_reference, i, str(file_name)) + for i, file_name in enumerate(inputfiles) if i != 0] + + completed = 1 + for future in as_completed(futures): + i, tform = future.result() + transforms[i] = tform + completed += 1 + + if completed == total_jobs or completed % 10 == 0: + log_info(f"Multiprocessing transformations progress: {completed}/{total_jobs}") + + return transforms + + +def log_alignment_progress(i, total_jobs, file_name, use_multiprocess_progress, + pixel_alignment_enabled=False): + if use_multiprocess_progress: + completed = i + 1 + if completed == total_jobs or completed % 10 == 0: + mode = "pixel alignment" if pixel_alignment_enabled else "WCS coordinate projection" + log_info(f"Multiprocessing {mode} progress: {completed}/{total_jobs}") + return + + display_file_name = _display_filename(file_name) + action = "Pixel-aligning" if pixel_alignment_enabled else "WCS-locating stars in" + message = f"{action} frame {i + 1} of {total_jobs} : {display_file_name}\n" + sys.stdout.write(message) + log.debug(message) + sys.stdout.flush() + + +def get_img_scale(hdr, wcs_file, pixel_init): + if wcs_file: + wcs_hdr = get_first_image_header(wcs_file) + astrometry_scale = [key.value.split(' ') for key in wcs_hdr._cards if 'scale:' in str(key.value)] + + if astrometry_scale: + img_scale_num = astrometry_scale[0][1] + img_scale_units = astrometry_scale[0][2] + else: + wcs = WCS(wcs_hdr).proj_plane_pixel_scales() + img_scale_num = (wcs[0].value + wcs[1].value) / 2 * 3600 # Convert to arcsec/pixel + img_scale_units = "arcsec/pixel" + elif 'IM_SCALE' in hdr: + img_scale_num = hdr['IM_SCALE'] + img_scale_units = hdr.comments['IM_SCALE'] + elif 'PIXSCALE' in hdr: + img_scale_num = hdr['PIXSCALE'] + img_scale_units = hdr.comments['PIXSCALE'] + elif pixel_init: + img_scale_num = pixel_init + img_scale_units = "arcsec/pixel" + else: + log_info("Not able to find Image Scale in the Image Header.") + img_scale_num = user_input("Please enter Image Scale (arcsec/pixel): ", type_=float) + img_scale_units = "arcsec/pixel" + + img_scale = f"Image scale in {img_scale_units}: {round_to_2(float(img_scale_num))}" + + return img_scale, float(img_scale_num) + + +def exp_time_med(exptimes): + # exposure time + consistent_et = False + if len(exptimes) > 0: + consistent_et = all(elem == exptimes[0] for elem in exptimes) + + exptimes = np.array(exptimes) + + if consistent_et: + return exptimes[0] + else: + return np.median(exptimes) + + +def update_coordinates_with_proper_motion(info_dict, time_obs): + parameter_names = { + 'dist': 'Distance (pc)', + 'pm_ra': 'Proper Motion RA (mas/yr)', + 'pm_dec': 'Proper Motion DEC (mas/yr)' + } + + numeric_values = {} + missing_values = [] + + for key in ['dist', 'pm_ra', 'pm_dec']: + raw_value = info_dict.get(key, 0.0) + + try: + parsed_value = float(raw_value) + except (TypeError, ValueError): + parsed_value = 0.0 + + numeric_values[key] = parsed_value + if parsed_value == 0.0: + missing_values.append(parameter_names[key]) + + if missing_values: + missing_values = ", ".join(missing_values) + log_info("Warning: Cannot account for proper motion due to missing values in: " + f"\n{missing_values}. If you find your target or comparisons are not detected well, please " + f"re-run and fill in values in the initialization file to account for proper motion", warn=True) + + return info_dict['ra'], info_dict['dec'] + else: + time_j2000 = Time(2000.0, format='jyear') + time_obs = Time(time_obs, format='jd') + + coord = SkyCoord( + ra=info_dict['ra'] * u.deg, + dec=info_dict['dec'] * u.deg, + distance=numeric_values['dist'] * u.pc, + pm_ra_cosdec=numeric_values['pm_ra'] * u.mas / u.yr, + pm_dec=numeric_values['pm_dec'] * u.mas / u.yr, + frame="icrs", + obstime=time_j2000 + ) + + updated_coord = coord.apply_space_motion(new_obstime=time_obs) + return updated_coord.ra.deg, updated_coord.dec.deg + + +def gaussian_psf(x, y, x0, y0, a, sigx, sigy, rot, b): + rx = (x - x0) * np.cos(rot) - (y - y0) * np.sin(rot) + ry = (x - x0) * np.sin(rot) + (y - y0) * np.cos(rot) + gausx = np.exp(-rx ** 2 / (2 * sigx ** 2)) + gausy = np.exp(-ry ** 2 / (2 * sigy ** 2)) + return a * gausx * gausy + b + + +def mesh_box(pos, box, maxx=0, maxy=0): + pos = [int(np.round(pos[0])), int(np.round(pos[1]))] + if maxx: + x = np.arange(max(0,pos[0] - box), min(maxx, pos[0] + box + 1)) + else: + x = np.arange(max(0,pos[0] - box), pos[0] + box + 1) + if maxy: + y = np.arange(max(0,pos[1] - box), min(maxy, pos[1] + box + 1)) + else: + y = np.arange(max(0,pos[1] - box), pos[1] + box + 1) + xv, yv = np.meshgrid(x, y) + return xv.astype(int), yv.astype(int) + + +def should_use_fast_centroid(frame_index): + return frame_index % CENTROID_FULL_FIT_CADENCE != 0 + + +def should_use_fast_target_centroid(frame_index, adaptive_apertures=False): + return should_use_fast_centroid(frame_index) and not adaptive_apertures + + +def _fit_centroid_moments(subarray, xv, yv, pos, box): + background = bn.nanmedian(subarray) + weights = subarray - background + weights = np.where(np.isfinite(weights) & (weights > 0), weights, 0.0) + wsum = np.sum(weights) + + if not np.isfinite(wsum) or wsum <= 0: + floor = np.nanmin(subarray) + weights = subarray - floor + weights = np.where(np.isfinite(weights) & (weights > 0), weights, 0.0) + wsum = np.sum(weights) + + if not np.isfinite(wsum) or wsum <= 0: + return np.empty(7) * np.nan + + wx = float(np.sum(xv * weights) / wsum) + wy = float(np.sum(yv * weights) / wsum) + + # Keep centroid near the expected star location in crowded fields. + wx = float(np.clip(wx, pos[0] - box * 0.5, pos[0] + box * 0.5)) + wy = float(np.clip(wy, pos[1] - box * 0.5, pos[1] + box * 0.5)) + + dx = xv - wx + dy = yv - wy + var_x = float(np.sum(weights * dx * dx) / wsum) + var_y = float(np.sum(weights * dy * dy) / wsum) + cov_xy = float(np.sum(weights * dx * dy) / wsum) + + sigx = float(np.clip(np.sqrt(max(var_x, 0.25)), 0.5, 20.0)) + sigy = float(np.clip(np.sqrt(max(var_y, 0.25)), 0.5, 20.0)) + rot = float(0.5 * np.arctan2(2.0 * cov_xy, var_x - var_y)) if np.isfinite(cov_xy) else 0.0 + amp = float(max(np.nanmax(subarray) - background, 0.0)) + + return np.array([wx, wy, amp, sigx, sigy, rot, float(background)], dtype=float) + + +def _has_usable_centroid_signal(subarray, amplitude, min_snr=5.0): + if not np.isfinite(amplitude) or amplitude <= 0: + return False + + scatter = float(bn.nanstd(subarray)) + if not np.isfinite(scatter) or scatter <= 0: + return True + + return amplitude >= (min_snr * scatter) + + +def _fit_seed_anchored_psf(data, pos, psf_function=gaussian_psf, box=8, bound_radius=4.0): + xv, yv = mesh_box(pos, box, maxx=data.shape[1], maxy=data.shape[0]) + subarray = data[yv, xv] + moment_fit = _fit_centroid_moments(subarray, xv, yv, pos, box) + + try: + init = [np.nanmax(subarray) - np.nanmin(subarray), 1.0, 1.0, 0.0, np.nanmin(subarray)] + except ValueError: + return _nan_psf_result() + + if np.isfinite(moment_fit[0]): + init = [ + moment_fit[2], + min(float(moment_fit[3]), 3.0), + min(float(moment_fit[4]), 3.0), + moment_fit[5], + moment_fit[6], + ] + + bound_radius = float(bound_radius) + sigma_upper = max(float(PSF_FIT_MAX_SIGMA_PIXELS), 0.5) + lo = [ + pos[0] - bound_radius, + pos[1] - bound_radius, + 0, + 0.5, + 0.5, + -np.pi / 4, + np.nanmin(subarray) - 1, + ] + up = [ + pos[0] + bound_radius, + pos[1] + bound_radius, + 1e7, + sigma_upper, + sigma_upper, + np.pi / 4, + np.nanmax(subarray) + 1, + ] + x0 = np.array([pos[0], pos[1], *init], dtype=float) + lo_arr = np.array(lo, dtype=float) + up_arr = np.array(up, dtype=float) + if np.all(np.isfinite(x0)): + x0 = np.clip(x0, lo_arr + 1e-6, up_arr - 1e-6) + + def fcn2min(pars): + model = psf_function(xv, yv, *pars) + return (subarray - model).flatten() + + res = least_squares(fcn2min, x0=x0, bounds=[lo, up], jac='2-point', xtol=None, method='trf') + if np.isfinite(moment_fit[6]): + res.x[6] = moment_fit[6] + return res.x + + +def _nan_psf_result(): + return np.full(7, np.nan, dtype=float) + + +def fit_centroid_or_warn_out_of_frame(data, pos, starIndex, **kwargs): + if not pixel_within_image(pos[0], pos[1], data.shape): + plateStatus.outOfFrameWarning(starIndex) + return _nan_psf_result() + return fit_centroid(data, pos, starIndex, **kwargs) + + +def psf_solution_quality_score(psf_row, seed_pos=None, + max_seed_offset_pixels=PSF_FIT_MAX_SEED_OFFSET_PIXELS, + max_axis_ratio=PSF_FIT_MAX_AXIS_RATIO, + max_sigma_pixels=PSF_FIT_MAX_SIGMA_PIXELS): + try: + row = np.asarray(psf_row, dtype=float).reshape(-1) + except (TypeError, ValueError): + return np.inf + if row.size < 5 or not np.all(np.isfinite(row[:5])): + return np.inf + + x_centroid, y_centroid, amplitude, sigma_x, sigma_y = row[:5] + if amplitude <= 0 or sigma_x <= 0 or sigma_y <= 0: + return np.inf + if max(sigma_x, sigma_y) > float(max_sigma_pixels): + return np.inf + + axis_ratio = max(sigma_x, sigma_y) / max(min(sigma_x, sigma_y), 1e-12) + if axis_ratio > float(max_axis_ratio): + return np.inf + + score = 0.25 * np.log(axis_ratio) + if seed_pos is not None: + try: + seed = np.asarray(seed_pos, dtype=float).reshape(-1) + except (TypeError, ValueError): + seed = np.array([], dtype=float) + if seed.size >= 2 and np.all(np.isfinite(seed[:2])): + seed_offset = float(np.hypot(x_centroid - seed[0], y_centroid - seed[1])) + if seed_offset > float(max_seed_offset_pixels): + return np.inf + score += seed_offset / max(float(max_seed_offset_pixels), 1e-12) + + mean_sigma = 0.5 * (sigma_x + sigma_y) + if np.isfinite(mean_sigma) and mean_sigma > 0: + score += 0.05 * abs(np.log(mean_sigma / 1.5)) + + score -= 0.005 * np.log1p(max(float(amplitude), 0.0)) + return float(score) + + +def choose_best_psf_solution(primary_row, fallback_row, seed_pos=None): + primary_score = psf_solution_quality_score(primary_row, seed_pos=seed_pos) + fallback_score = psf_solution_quality_score(fallback_row, seed_pos=seed_pos) + if np.isfinite(primary_score) and ( + not np.isfinite(fallback_score) + or primary_score <= fallback_score + PSF_FIT_SELECTION_MARGIN + ): + return np.asarray(primary_row, dtype=float) + if np.isfinite(fallback_score): + return np.asarray(fallback_row, dtype=float) + return _nan_psf_result() + + +def fractional_flux_change_within_limit(current_amplitude, previous_amplitude, limit=0.5): + if (not np.isfinite(current_amplitude) + or not np.isfinite(previous_amplitude) + or previous_amplitude == 0): + return False + + return np.abs((current_amplitude - previous_amplitude) / previous_amplitude) <= limit + + +def centroid_position_is_finite(psf_row): + try: + coords = np.asarray(psf_row[:2], dtype=float) + except (TypeError, ValueError, IndexError): + return False + + return bool(np.all(np.isfinite(coords))) + + +def choose_centroid_seed_position(predicted_pos, previous_psf_row=None, max_offset_pixels=5.0): + predicted = np.asarray(predicted_pos, dtype=float).reshape(-1) + if predicted.size < 2 or not np.all(np.isfinite(predicted[:2])): + return np.array([np.nan, np.nan], dtype=float) + + if not centroid_position_is_finite(previous_psf_row): + return np.array(predicted[:2], dtype=float) + + previous = np.asarray(previous_psf_row[:2], dtype=float) + if np.hypot(*(previous - predicted[:2])) > float(max_offset_pixels): + return np.array(predicted[:2], dtype=float) + + return previous.astype(float, copy=True) + + +def centroid_offset_matches_reference(psf_a, psf_b, expected_dx, expected_dy, + tolerance=WCS_REFERENCE_GEOMETRY_TOLERANCE_PIXELS): + if not centroid_position_is_finite(psf_a) or not centroid_position_is_finite(psf_b): + return False + if not np.isfinite(expected_dx) or not np.isfinite(expected_dy): + return False + + dx = float(abs(float(psf_a[0]) - float(psf_b[0]))) + dy = float(abs(float(psf_a[1]) - float(psf_b[1]))) + tolerance = float(tolerance) + + return ( + abs(dx - float(expected_dx)) <= tolerance + and abs(dy - float(expected_dy)) <= tolerance + ) + + +def should_keep_header_wcs_alignment( + projected_off_frame, + frame_index, + target_psf_row, + previous_target_psf_row=None, + comp_psf_rows=None, + previous_comp_psf_rows=None, + expected_offsets=None, + tolerance=WCS_REFERENCE_GEOMETRY_TOLERANCE_PIXELS, + min_geometry_match_fraction=WCS_MIN_GEOMETRY_MATCH_FRACTION, +): + decision = { + 'use_wcs_alignment': False, + 'reason': 'missing_target_centroid', + 'target_flux_change_ok': True, + 'comp_flux_change_ok': True, + 'geometry_match_count': 0, + 'geometry_test_count': 0, + } + + if projected_off_frame: + decision.update(use_wcs_alignment=True, reason='projected_off_frame') + return decision + + if not centroid_position_is_finite(target_psf_row): + return decision + + if frame_index == 0: + decision.update(use_wcs_alignment=True, reason='first_frame') + return decision + + if previous_target_psf_row is not None: + decision['target_flux_change_ok'] = fractional_flux_change_within_limit( + target_psf_row[2], + previous_target_psf_row[2], + ) + + comp_psf_rows = {} if comp_psf_rows is None else dict(comp_psf_rows) + previous_comp_psf_rows = {} if previous_comp_psf_rows is None else dict(previous_comp_psf_rows) + expected_offsets = {} if expected_offsets is None else dict(expected_offsets) + + for key, comp_row in comp_psf_rows.items(): + prev_comp_row = previous_comp_psf_rows.get(key) + if prev_comp_row is not None: + decision['comp_flux_change_ok'] = ( + decision['comp_flux_change_ok'] + and fractional_flux_change_within_limit(comp_row[2], prev_comp_row[2]) + ) + + if not centroid_position_is_finite(comp_row): + continue + + expected_offset = expected_offsets.get(key) + if expected_offset is None: + continue + + try: + expected_dx = float(expected_offset[0]) + expected_dy = float(expected_offset[1]) + except (TypeError, ValueError, IndexError): + continue + + decision['geometry_test_count'] += 1 + if centroid_offset_matches_reference( + comp_row, + target_psf_row, + expected_dx, + expected_dy, + tolerance=tolerance, + ): + decision['geometry_match_count'] += 1 + + geometry_test_count = decision['geometry_test_count'] + if geometry_test_count == 0: + decision.update(use_wcs_alignment=True, reason='finite_target_only') + return decision + + minimum_matches = max(1, int(np.ceil(float(min_geometry_match_fraction) * geometry_test_count))) + if decision['geometry_match_count'] >= minimum_matches: + decision.update(use_wcs_alignment=True, reason='geometry_match') + else: + decision['reason'] = 'geometry_mismatch' + + return decision + + +# Method fits a 2D gaussian function that matches the star_psf to the star image and returns its pixel coordinates +def fit_centroid(data, pos, starIndex, psf_function=gaussian_psf, box=15, weightedcenter=False, fast_mode=False): + stage_start = perf_counter() + # get sub field in image + try: + xv, yv = mesh_box(pos, box, maxx=data.shape[1], maxy=data.shape[0]) + subarray = data[yv, xv] + try: + init = [np.nanmax(subarray) - np.nanmin(subarray), 1, 1, 0, np.nanmin(subarray)] + except ValueError as ve: + # Handle null subfield - cannot solve + plateStatus.outOfFrameWarning(starIndex) + log.debug(f"Warning: empty subfield for fit_centroid at {np.round(pos, 2)}") + return _nan_psf_result() + + moment_fit = _fit_centroid_moments(subarray, xv, yv, pos, box) + if np.isfinite(moment_fit[0]): + wx, wy = moment_fit[0], moment_fit[1] + init = [moment_fit[2], moment_fit[3], moment_fit[4], moment_fit[5], moment_fit[6]] + if fast_mode: + if ( + _has_usable_centroid_signal(subarray, init[0]) + and centroid_position_is_finite(moment_fit) + ): + return moment_fit + + plateStatus.lowFluxAmplitudeWarning(starIndex, pos[0], pos[1]) + log.debug( + f"Warning: Measured fast centroid amplitude is really low---" + f"are you sure there is a star at {np.round(pos, 2)}?" + ) + return _nan_psf_result() + else: + # compute flux weighted centroid in x and y + wx = np.sum(xv[0] * subarray.sum(0)) / subarray.sum(0).sum() + wy = np.sum(yv[:, 0] * subarray.sum(1)) / subarray.sum(1).sum() + + # lower bound: [xc, yc, amp, sigx, sigy, rotation, bg] + lo = [pos[0] - box * 0.5, pos[1] - box * 0.5, 0, 0.5, 0.5, -np.pi / 4, np.nanmin(subarray) - 1] + up = [pos[0] + box * 0.5, pos[1] + box * 0.5, 1e7, 20, 20, np.pi / 4, np.nanmax(subarray) + 1] + x0 = np.array([*pos, *init], dtype=float) + lo_arr = np.array(lo, dtype=float) + up_arr = np.array(up, dtype=float) + if np.all(np.isfinite(x0)): + x0 = np.clip(x0, lo_arr + 1e-6, up_arr - 1e-6) + has_usable_signal = _has_usable_centroid_signal(subarray, init[0]) + + def fcn2min(pars): + model = psf_function(xv, yv, *pars) + return (subarray - model).flatten() + + try: + res = least_squares(fcn2min, x0=x0, bounds=[lo, up], jac='2-point', xtol=None, method='trf') + except Exception as exc: + if ( + has_usable_signal + and centroid_position_is_finite(moment_fit) + ): + log.debug(f"Centroid PSF fit failed at {np.round(pos, 2)}; using moment centroid instead: {exc}") + return moment_fit + + if not has_usable_signal: + plateStatus.lowFluxAmplitudeWarning(starIndex, pos[0], pos[1]) + log.debug(f"Warning: Measured flux amplitude is really low---are you sure there is a star at {np.round(pos, 2)}?") + else: + log.debug(f"Centroid PSF fit failed at {np.round(pos, 2)}; attempting LM fallback: {exc}") + + try: + res = least_squares(fcn2min, x0=x0, jac='2-point', xtol=1e-12, method='lm') + except Exception as lm_exc: + log.debug(f"Centroid LM fallback failed at {np.round(pos, 2)}: {lm_exc}") + return _nan_psf_result() + + # Preserve the solved PSF center for subpixel tracking by default. + # The weighted-center override remains available as an explicit legacy option. + if weightedcenter: + res.x[0] = wx + res.x[1] = wy + if np.isfinite(moment_fit[6]): + res.x[6] = moment_fit[6] + + anchored_fit = _nan_psf_result() + if not weightedcenter: + try: + anchored_box = max(4, min(int(box), 8)) + anchored_fit = _fit_seed_anchored_psf( + data, + pos, + psf_function=psf_function, + box=anchored_box, + bound_radius=4.0, + ) + except Exception as exc: + log.debug(f"Seed-anchored PSF fit failed at {np.round(pos, 2)}: {exc}") + + selected_fit = choose_best_psf_solution(res.x, anchored_fit, seed_pos=pos) + selected_fit = choose_best_psf_solution(selected_fit, moment_fit, seed_pos=pos) + if not np.all(np.isfinite(selected_fit[:5])): + log.debug( + f"Centroid PSF fit at {np.round(pos, 2)} rejected as implausible " + "(large seed offset, elongated PSF, or invalid width)." + ) + return selected_fit + finally: + _record_photometry_stage_timing('fit_centroid', perf_counter() - stage_start) + + +def fit_psf_photometry_flux_row(data, centroid_row, starIndex, psf_function=gaussian_psf, box=15): + try: + centroid_row = np.asarray(centroid_row, dtype=float).reshape(-1) + except (TypeError, ValueError): + return _nan_psf_result() + if centroid_row.size < 2 or not centroid_position_is_finite(centroid_row): + return _nan_psf_result() + + pos = centroid_row[:2] + try: + xv, yv = mesh_box(pos, box, maxx=data.shape[1], maxy=data.shape[0]) + subarray = data[yv, xv] + init = [ + np.nanmax(subarray) - np.nanmin(subarray), + 1.0, + 1.0, + 0.0, + np.nanmin(subarray), + ] + except Exception: + return centroid_row.copy() if centroid_row.size >= 7 else _nan_psf_result() + + lo = [ + pos[0] - box * 0.5, + pos[1] - box * 0.5, + 0, + 0.5, + 0.5, + -np.pi / 4, + np.nanmin(subarray) - 1, + ] + up = [ + pos[0] + box * 0.5, + pos[1] + box * 0.5, + 1e7, + 20, + 20, + np.pi / 4, + np.nanmax(subarray) + 1, + ] + + def fcn2min(pars): + model = psf_function(xv, yv, *pars) + return (subarray - model).flatten() + + try: + res = least_squares( + fcn2min, + x0=[*pos, *init], + bounds=[lo, up], + jac='3-point', + xtol=None, + method='trf', + ) + flux_row = np.asarray(res.x, dtype=float) + except Exception as exc: + log.debug( + f"Stable PSF photometry flux fit failed at {np.round(pos, 2)} for star {starIndex}; " + f"using centroid fit flux parameters instead: {exc}" + ) + return centroid_row.copy() if centroid_row.size >= 7 else _nan_psf_result() + + if not np.all(np.isfinite(flux_row[:5])): + return centroid_row.copy() if centroid_row.size >= 7 else _nan_psf_result() + if not np.isfinite(psf_solution_quality_score(flux_row, seed_pos=pos)): + return centroid_row.copy() if centroid_row.size >= 7 else _nan_psf_result() + + # Keep the robust centroid/offset solution for diagnostics and aperture placement, + # but use the legacy-stable Gaussian amplitude/width for PSF flux integration. + flux_row[:2] = centroid_row[:2] + return flux_row + + +def fit_legacy_psf_photometry_flux_row(data, centroid_row, starIndex, psf_function=gaussian_psf, box=15): + try: + centroid_row = np.asarray(centroid_row, dtype=float).reshape(-1) + except (TypeError, ValueError): + return _nan_psf_result() + if centroid_row.size < 2 or not centroid_position_is_finite(centroid_row): + return _nan_psf_result() + + pos = centroid_row[:2] + try: + xv, yv = mesh_box(pos, box, maxx=data.shape[1], maxy=data.shape[0]) + subarray = data[yv, xv] + init = [ + np.nanmax(subarray) - np.nanmin(subarray), + 1, + 1, + 0, + np.nanmin(subarray), + ] + except ValueError: + plateStatus.outOfFrameWarning(starIndex) + log.debug(f"Warning: empty subfield for legacy PSF flux fit at {np.round(pos, 2)}") + return centroid_row.copy() if centroid_row.size >= 7 else _nan_psf_result() + except Exception as exc: + log.debug(f"Legacy PSF flux setup failed at {np.round(pos, 2)} for star {starIndex}: {exc}") + return centroid_row.copy() if centroid_row.size >= 7 else _nan_psf_result() + + try: + wx = np.sum(xv[0] * subarray.sum(0)) / subarray.sum(0).sum() + wy = np.sum(yv[:, 0] * subarray.sum(1)) / subarray.sum(1).sum() + except Exception: + wx, wy = pos[0], pos[1] + + lo = [ + pos[0] - box * 0.5, + pos[1] - box * 0.5, + 0, + 0.5, + 0.5, + -np.pi / 4, + np.nanmin(subarray) - 1, + ] + up = [ + pos[0] + box * 0.5, + pos[1] + box * 0.5, + 1e7, + 20, + 20, + np.pi / 4, + np.nanmax(subarray) + 1, + ] + + def fcn2min(pars): + model = psf_function(xv, yv, *pars) + return (subarray - model).flatten() + + try: + res = least_squares( + fcn2min, + x0=[*pos, *init], + bounds=[lo, up], + jac='3-point', + xtol=None, + method='trf', + ) + except Exception as exc: + plateStatus.lowFluxAmplitudeWarning(starIndex, pos[0], pos[1]) + log.debug( + f"Legacy PSF flux fit failed at {np.round(pos, 2)} for star {starIndex}; " + f"trying unbounded LM fallback: {exc}" + ) + try: + res = least_squares( + fcn2min, + x0=[*pos, *init], + jac='3-point', + xtol=None, + method='lm', + ) + except Exception as lm_exc: + log.debug( + f"Legacy PSF flux LM fallback failed at {np.round(pos, 2)} for star {starIndex}; " + f"using centroid fit flux parameters instead: {lm_exc}" + ) + return centroid_row.copy() if centroid_row.size >= 7 else _nan_psf_result() + + flux_row = np.asarray(res.x, dtype=float) + if flux_row.size >= 2: + flux_row[0] = wx + flux_row[1] = wy + return flux_row + + +def _psf_seed_track_candidate_paths(seed_track_directory, key): + root = Path(seed_track_directory).expanduser() + search_dirs = [root] + if root.name.lower() not in {'working_artifacts', 'temp'}: + search_dirs.extend((root / 'working_artifacts', root / 'temp')) + + expanded_search_dirs = [] + for directory in search_dirs: + expanded_search_dirs.append(directory) + expanded_search_dirs.append(directory / 'psf_flux_data') + search_dirs = expanded_search_dirs + + if key == 'target': + names = ( + 'psf_data_target.txt', + 'psf_flux_data_target.txt', + ) + else: + comp_index = key[4:] if key.startswith('comp') else '' + names = ( + f'psf_data_{key}.txt', + f'psf_flux_data_{key}.txt', + f'psf_data_comp{comp_index}.txt' if comp_index else '', + f'psf_flux_data_comp{comp_index}.txt' if comp_index else '', + 'psf_data_comp.txt' if key == 'comp1' else '', + 'psf_flux_data_comp.txt' if key == 'comp1' else '', + ) + + for directory in search_dirs: + for name in names: + if not name: + continue + path = directory / name + if path.exists(): + return path + return None + + +def _load_psf_seed_track_file(path, expected_frame_count, key): + try: + rows = np.loadtxt(path, comments='#') + except Exception as exc: + log_info( + f"Warning: Could not load PSF seed track for {key} from {path}: {exc}", + warn=True, + ) + return None + + rows = np.asarray(rows, dtype=float) + if rows.ndim == 1: + rows = rows.reshape(1, -1) + + if rows.ndim != 2 or rows.shape[1] < 7: + log_info( + f"Warning: Ignoring PSF seed track for {key}; expected at least 7 columns in {path}.", + warn=True, + ) + return None + + if rows.shape[0] != int(expected_frame_count): + log_info( + f"Warning: Ignoring PSF seed track for {key}; {path} has {rows.shape[0]} row(s), " + f"but the reduction has {int(expected_frame_count)} frame(s).", + warn=True, + ) + return None + + return np.array(rows[:, :7], dtype=float, copy=True) + + +def load_psf_flux_seed_tracks(seed_track_directory, expected_frame_count, comp_alignment_keys): + seed_track_directory = psf_seed_track_directory_from_config(seed_track_directory) + if seed_track_directory is None: + return {} + + seed_tracks = {} + for key in ('target', *tuple(comp_alignment_keys or ())): + path = _psf_seed_track_candidate_paths(seed_track_directory, key) + if path is None: + log_info( + f"Warning: PSF seed track directory {seed_track_directory} did not contain a seed file for {key}.", + warn=True, + ) + continue + rows = _load_psf_seed_track_file(path, expected_frame_count, key) + if rows is not None: + seed_tracks[key] = rows + + if seed_tracks: + loaded_keys = ', '.join(seed_tracks) + log_info( + f"Loaded PSF flux seed track row(s) from {seed_track_directory} for: {loaded_keys}. " + "These rows will seed PSF flux fits only; alignment centroids remain unchanged." + ) + else: + log_info( + f"Warning: No usable PSF flux seed track rows were loaded from {seed_track_directory}.", + warn=True, + ) + + return seed_tracks + + +def sigma_clipped_nanmedian(data, sigma=3.0, max_iters=3): + clipped = np.array(data, dtype=float, copy=True) + if clipped.size == 0: + return np.nan, np.nan + + clipped[~np.isfinite(clipped)] = np.nan + nan_count_prev = np.count_nonzero(np.isnan(clipped)) + + for _ in range(max_iters): + center = bn.nanmedian(clipped) + scatter = bn.nanstd(clipped) + if not np.isfinite(center): + return np.nan, np.nan + if not np.isfinite(scatter) or scatter <= 0: + break + + clipped[np.abs(clipped - center) > sigma * scatter] = np.nan + nan_count = np.count_nonzero(np.isnan(clipped)) + if nan_count == nan_count_prev: + break + nan_count_prev = nan_count + + return bn.nanmedian(clipped), bn.nanstd(clipped) + + +def normalize_flux_series_to_approximate_unity( + flux_values, + flux_errors=None, + sigma=3.0, + max_iters=3, + min_points=LIGHTCURVE_MIN_VALID_POINTS, +): + flux_values = np.asarray(flux_values, dtype=float) + normalized_flux = np.array(flux_values, dtype=float, copy=True) + normalized_unc = None if flux_errors is None else np.array(flux_errors, dtype=float, copy=True) + + valid_flux = np.isfinite(flux_values) & (flux_values > 0) + if np.count_nonzero(valid_flux) < max(1, int(min_points)): + return normalized_flux, normalized_unc, np.nan + + baseline_level, _ = sigma_clipped_nanmedian( + flux_values[valid_flux], + sigma=sigma, + max_iters=max_iters, + ) + if not np.isfinite(baseline_level) or baseline_level <= 0: + baseline_level = bn.nanmedian(flux_values[valid_flux]) + if not np.isfinite(baseline_level) or baseline_level <= 0: + return normalized_flux, normalized_unc, np.nan + + normalized_flux[valid_flux] = flux_values[valid_flux] / baseline_level + if normalized_unc is not None and normalized_unc.shape == flux_values.shape: + finite_unc = np.isfinite(normalized_unc) + normalized_unc[finite_unc] = normalized_unc[finite_unc] / baseline_level + + return normalized_flux, normalized_unc, float(baseline_level) + + +def is_blank_noise_budget_value(value): + if value is None: + return True + if isinstance(value, str): + return value.strip() == "" + return False + + +def coerce_noise_budget_scalar(value, *, require_positive=False, require_nonnegative=False): + if is_blank_noise_budget_value(value): + return np.nan + try: + scalar = float(value) + except (TypeError, ValueError): + return np.nan + if not np.isfinite(scalar): + return np.nan + if require_positive and scalar <= 0: + return np.nan + if require_nonnegative and scalar < 0: + return np.nan + return float(scalar) + + +def noise_budget_value_from_mapping(mapping, keys, *, require_positive=False, require_nonnegative=False): + if not isinstance(mapping, dict): + return np.nan + for key in keys: + if key not in mapping: + continue + value = coerce_noise_budget_scalar( + mapping.get(key), + require_positive=require_positive, + require_nonnegative=require_nonnegative, + ) + if np.isfinite(value): + return value + return np.nan + + +def noise_budget_value_from_header(header, keys, *, require_positive=False, require_nonnegative=False): + if header is None: + return np.nan + try: + header_keys = set(header.keys()) + except AttributeError: + return np.nan + upper_lookup = {str(key).upper(): key for key in header_keys} + for key in keys: + actual_key = upper_lookup.get(str(key).upper()) + if actual_key is None: + continue + value = coerce_noise_budget_scalar( + header.get(actual_key), + require_positive=require_positive, + require_nonnegative=require_nonnegative, + ) + if np.isfinite(value): + return value + return np.nan + + +def resolve_noise_budget_value(info_dict, aliases, header=None, header_keys=(), + *, require_positive=False, require_nonnegative=False): + value = noise_budget_value_from_mapping( + info_dict, + aliases, + require_positive=require_positive, + require_nonnegative=require_nonnegative, + ) + if np.isfinite(value): + return value, 'inits' + value = noise_budget_value_from_header( + header, + header_keys, + require_positive=require_positive, + require_nonnegative=require_nonnegative, + ) + if np.isfinite(value): + return value, 'fits_header' + return np.nan, None + + +def noise_budget_config_from_info(info_dict, header=None): + info_dict = info_dict if isinstance(info_dict, dict) else {} + gain, gain_source = resolve_noise_budget_value( + info_dict, + ( + 'gain_electrons_per_adu', + 'gain_e_per_adu', + 'gain', + 'Gain (e-/ADU)', + 'CCD Gain (e-/ADU)', + ), + header=header, + header_keys=NOISE_GAIN_HEADER_KEYS, + require_positive=True, + ) + if not np.isfinite(gain) or gain <= 0: + gain = 1.0 + gain_source = 'default' + + read_noise, read_source = resolve_noise_budget_value( + info_dict, + ( + 'read_noise_electrons', + 'read_noise_e', + 'read_noise', + 'Read Noise (e-)', + 'CCD Read Noise (e-)', + ), + header=header, + header_keys=NOISE_READ_HEADER_KEYS, + require_nonnegative=True, + ) + dark_current, dark_source = resolve_noise_budget_value( + info_dict, + ( + 'dark_current_electrons_per_second_per_pixel', + 'dark_current_e_per_s_pix', + 'dark_current', + 'Dark Current (e-/s/pix)', + ), + header=header, + header_keys=NOISE_DARK_HEADER_KEYS, + require_nonnegative=True, + ) + flat_fraction, flat_source = resolve_noise_budget_value( + info_dict, + ( + 'flat_field_fractional_error', + 'flat_field_fractional_noise', + 'flat_field_error_fraction', + 'Flat Field Fractional Error', + 'Flat-Field Fractional Error', + ), + header=header, + header_keys=NOISE_FLAT_HEADER_KEYS, + require_nonnegative=True, + ) + scintillation_coefficient, scint_source = resolve_noise_budget_value( + info_dict, + ( + 'scintillation_coefficient', + 'scintillation_noise_coefficient', + 'Scintillation Coefficient', + ), + header=header, + header_keys=NOISE_SCINTILLATION_HEADER_KEYS, + require_positive=True, + ) + telescope_aperture_m, aperture_source = resolve_noise_budget_value( + info_dict, + ( + 'telescope_aperture_m', + 'telescope_aperture_meters', + 'Telescope Aperture (m)', + ), + header=header, + header_keys=NOISE_APERTURE_HEADER_KEYS, + require_positive=True, + ) + telescope_aperture_cm, aperture_cm_source = resolve_noise_budget_value( + info_dict, + ( + 'telescope_aperture_cm', + 'Telescope Aperture (cm)', + ), + require_positive=True, + ) + telescope_aperture_mm, aperture_mm_source = resolve_noise_budget_value( + info_dict, + ( + 'telescope_aperture_mm', + 'Telescope Aperture (mm)', + ), + header=header, + header_keys=NOISE_APERTURE_MM_HEADER_KEYS, + require_positive=True, + ) + if np.isfinite(telescope_aperture_cm) and telescope_aperture_cm > 0: + telescope_aperture_m = telescope_aperture_cm / 100.0 + aperture_source = aperture_cm_source + elif np.isfinite(telescope_aperture_mm) and telescope_aperture_mm > 0: + telescope_aperture_m = telescope_aperture_mm / 1000.0 + aperture_source = aperture_mm_source + if not np.isfinite(scintillation_coefficient): + scintillation_coefficient = SCINTILLATION_COEFFICIENT_DEFAULT + scint_source = 'default' + + enabled_terms = ['source', 'sky_aperture', 'sky_estimate'] + source_by_term = {'gain': gain_source} + optional_terms = { + 'read': (read_noise, read_source), + 'dark': (dark_current, dark_source), + 'flat': (flat_fraction, flat_source), + } + for term, (value, source) in optional_terms.items(): + if np.isfinite(value) and value > 0: + enabled_terms.append(term) + source_by_term[term] = source + if np.isfinite(telescope_aperture_m) and telescope_aperture_m > 0: + enabled_terms.append('scintillation') + source_by_term['scintillation'] = aperture_source + source_by_term['scintillation_coefficient'] = scint_source + + return { + 'gain_e_per_adu': float(gain), + 'read_noise_electrons': read_noise, + 'dark_current_electrons_per_second_per_pixel': dark_current, + 'flat_field_fractional_error': flat_fraction, + 'scintillation_coefficient': scintillation_coefficient, + 'telescope_aperture_m': telescope_aperture_m, + 'elevation_m': coerce_noise_budget_scalar(info_dict.get('elev'), require_nonnegative=True), + 'enabled_terms': tuple(enabled_terms), + 'source_by_term': source_by_term, + } + + +def format_noise_budget_config_summary(config): + if not isinstance(config, dict): + return "source, sky_aperture, sky_estimate" + parts = [] + gain = config.get('gain_e_per_adu', 1.0) + parts.append(f"gain={gain:.4g} e-/ADU") + for term in ('read', 'dark', 'flat', 'scintillation'): + if term not in config.get('enabled_terms', ()): + continue + if term == 'read': + parts.append(f"read={config.get('read_noise_electrons'):.4g} e-") + elif term == 'dark': + parts.append( + f"dark={config.get('dark_current_electrons_per_second_per_pixel'):.4g} e-/s/pix" + ) + elif term == 'flat': + parts.append(f"flat={config.get('flat_field_fractional_error'):.4g} frac") + elif term == 'scintillation': + parts.append(f"scintillation D={config.get('telescope_aperture_m'):.4g} m") + return ", ".join(parts) + + +def empty_noise_budget_grids(shape): + return { + component: np.full(shape, np.nan, dtype=float) + for component in NOISE_BUDGET_COMPONENT_KEYS + } + + +def empty_noise_budget_series(length): + return { + component: np.full(int(length), np.nan, dtype=float) + for component in NOISE_BUDGET_COMPONENT_KEYS + } + + +def compute_scintillation_fraction(config, exposure_s=np.nan, airmass=np.nan): + if not isinstance(config, dict) or 'scintillation' not in config.get('enabled_terms', ()): + return np.nan + aperture_m = coerce_noise_budget_scalar(config.get('telescope_aperture_m'), require_positive=True) + exposure_s = coerce_noise_budget_scalar(exposure_s, require_positive=True) + if not np.isfinite(aperture_m) or not np.isfinite(exposure_s): + return np.nan + airmass = coerce_noise_budget_scalar(airmass, require_positive=True) + if not np.isfinite(airmass): + airmass = 1.0 + elevation_m = coerce_noise_budget_scalar(config.get('elevation_m'), require_nonnegative=True) + if not np.isfinite(elevation_m): + elevation_m = 0.0 + coefficient = coerce_noise_budget_scalar( + config.get('scintillation_coefficient', SCINTILLATION_COEFFICIENT_DEFAULT), + require_positive=True, + ) + if not np.isfinite(coefficient): + coefficient = SCINTILLATION_COEFFICIENT_DEFAULT + aperture_cm = aperture_m * 100.0 + return float( + coefficient + * aperture_cm ** (-2.0 / 3.0) + * airmass ** 1.75 + * np.exp(-elevation_m / 8000.0) + / np.sqrt(2.0 * exposure_s) + ) + + +def compute_photometry_noise_budget(flux_adu, sky_sigma_adu, aperture_pixels, sky_pixels, + exposure_s=np.nan, airmass=np.nan, noise_config=None): + config = noise_config if isinstance(noise_config, dict) else {} + gain = coerce_noise_budget_scalar(config.get('gain_e_per_adu', 1.0), require_positive=True) + if not np.isfinite(gain): + gain = 1.0 + flux_adu = coerce_noise_budget_scalar(flux_adu) + sky_sigma_adu = coerce_noise_budget_scalar(sky_sigma_adu, require_nonnegative=True) + aperture_pixels = coerce_noise_budget_scalar(aperture_pixels, require_positive=True) + sky_pixels = coerce_noise_budget_scalar(sky_pixels, require_positive=True) + exposure_s = coerce_noise_budget_scalar(exposure_s, require_positive=True) + + variances = {component: 0.0 for component in NOISE_BUDGET_COMPONENT_KEYS if component != 'total'} + if np.isfinite(flux_adu): + variances['source'] = max(float(flux_adu), 0.0) / gain + if np.isfinite(sky_sigma_adu) and np.isfinite(aperture_pixels): + variances['sky_aperture'] = aperture_pixels * sky_sigma_adu ** 2 + if np.isfinite(sky_pixels) and sky_pixels > 0: + variances['sky_estimate'] = ( + NOISE_BUDGET_SKY_MEDIAN_VARIANCE_FACTOR + * aperture_pixels ** 2 + * sky_sigma_adu ** 2 + / sky_pixels + ) + + read_noise = coerce_noise_budget_scalar(config.get('read_noise_electrons'), require_nonnegative=True) + if np.isfinite(read_noise) and np.isfinite(aperture_pixels): + variances['read'] = aperture_pixels * (read_noise / gain) ** 2 + + dark_current = coerce_noise_budget_scalar( + config.get('dark_current_electrons_per_second_per_pixel'), + require_nonnegative=True, + ) + if np.isfinite(dark_current) and np.isfinite(exposure_s) and np.isfinite(aperture_pixels): + variances['dark'] = aperture_pixels * dark_current * exposure_s / (gain ** 2) + + flux_abs = abs(float(flux_adu)) if np.isfinite(flux_adu) else np.nan + flat_fraction = coerce_noise_budget_scalar( + config.get('flat_field_fractional_error'), + require_nonnegative=True, + ) + if np.isfinite(flat_fraction) and np.isfinite(flux_abs): + variances['flat'] = (flat_fraction * flux_abs) ** 2 + + scintillation_fraction = compute_scintillation_fraction(config, exposure_s=exposure_s, airmass=airmass) + if np.isfinite(scintillation_fraction) and np.isfinite(flux_abs): + variances['scintillation'] = (scintillation_fraction * flux_abs) ** 2 + + total_variance = float( + np.nansum([ + variance + for variance in variances.values() + if np.isfinite(variance) and variance >= 0 + ]) + ) + budget = {} + for component in NOISE_BUDGET_COMPONENT_KEYS: + variance = total_variance if component == 'total' else variances.get(component, 0.0) + budget[component] = float(np.sqrt(max(variance, 0.0))) if np.isfinite(variance) else np.nan + return budget + + +def valid_flux_error_array(flux_error, shape): + if flux_error is None: + return None + try: + values = np.asarray(flux_error, dtype=float) + except (TypeError, ValueError): + return None + if values.shape != shape: + return None + return values + + +def relative_flux_uncertainty_from_star_errors(target_flux, comp_flux, + target_flux_error=None, comp_flux_error=None): + target_flux = np.asarray(target_flux, dtype=float) + comp_flux = np.asarray(comp_flux, dtype=float) + target_flux_error = valid_flux_error_array(target_flux_error, target_flux.shape) + comp_flux_error = valid_flux_error_array(comp_flux_error, comp_flux.shape) + + with np.errstate(invalid='ignore'): + target_fallback_error = np.sqrt(target_flux) + comp_fallback_error = np.sqrt(comp_flux) + + if target_flux_error is not None: + valid_target_error = np.isfinite(target_flux_error) & (target_flux_error > 0) + target_sigma = np.where(valid_target_error, target_flux_error, target_fallback_error) + else: + target_sigma = target_fallback_error + + if comp_flux_error is not None: + valid_comp_error = np.isfinite(comp_flux_error) & (comp_flux_error > 0) + comp_sigma = np.where(valid_comp_error, comp_flux_error, comp_fallback_error) + else: + comp_sigma = comp_fallback_error + + if np.allclose(comp_flux, 1.0): + return target_sigma + + with np.errstate(divide='ignore', invalid='ignore'): + return np.sqrt( + (target_sigma / comp_flux) ** 2 + + (comp_sigma * target_flux / comp_flux ** 2) ** 2 + ) + + +def exposure_scale_factors_to_max(exposure_times_seconds): + if exposure_times_seconds is None: + return None + try: + exposure_times = np.asarray(exposure_times_seconds, dtype=float) + except (TypeError, ValueError): + return None + if exposure_times.ndim != 1: + return None + + valid = np.isfinite(exposure_times) & (exposure_times > 0) + if not np.any(valid): + return None + + max_exposure = float(np.nanmax(exposure_times[valid])) + if not np.isfinite(max_exposure) or max_exposure <= 0: + return None + + factors = np.ones(exposure_times.shape, dtype=float) + factors[valid] = max_exposure / exposure_times[valid] + return factors + + +def source_flux_uncertainty_from_counts(flux_adu, gain_e_per_adu=None): + try: + gain = float(gain_e_per_adu) + except (TypeError, ValueError): + gain = 1.0 + if not np.isfinite(gain) or gain <= 0: + gain = 1.0 + + flux_adu = np.asarray(flux_adu, dtype=float) + with np.errstate(invalid='ignore'): + return np.sqrt(np.maximum(flux_adu, 0.0) / gain) + + +def scale_target_only_flux_to_common_exposure(target_flux, target_flux_error, comp_flux, + exposure_times_seconds=None, gain_e_per_adu=None): + target_flux = np.asarray(target_flux, dtype=float) + comp_flux = np.asarray(comp_flux, dtype=float) + target_flux_error = valid_flux_error_array(target_flux_error, target_flux.shape) + + if target_flux.ndim != 1 or comp_flux.shape != target_flux.shape: + return target_flux, target_flux_error + if not np.allclose(comp_flux, 1.0, equal_nan=False): + return target_flux, target_flux_error + + scale_factors = exposure_scale_factors_to_max(exposure_times_seconds) + if scale_factors is None or scale_factors.shape != target_flux.shape: + return target_flux, target_flux_error + if np.allclose(scale_factors, 1.0, rtol=1e-12, atol=1e-12): + return target_flux, target_flux_error + + scaled_flux = target_flux * scale_factors + fallback_error = source_flux_uncertainty_from_counts(target_flux, gain_e_per_adu) + if target_flux_error is None: + scaled_error = fallback_error * scale_factors + else: + valid_error = np.isfinite(target_flux_error) & (target_flux_error > 0) + scaled_error = np.where(valid_error, target_flux_error, fallback_error) * scale_factors + return scaled_flux, scaled_error + + +def weighted_nanpercentile(values, weights, percentile): + values = np.asarray(values, dtype=float).ravel() + weights = np.asarray(weights, dtype=float).ravel() + valid = np.isfinite(values) & np.isfinite(weights) & (weights > 0) + if not np.any(valid): + return np.nan + + values = values[valid] + weights = weights[valid] + sort_index = np.argsort(values, kind='mergesort') + values = values[sort_index] + weights = weights[sort_index] + + total_weight = float(np.sum(weights)) + if not np.isfinite(total_weight) or total_weight <= 0: + return np.nan + + if values.size == 1: + return float(values[0]) + + cumulative = (np.cumsum(weights) - 0.5 * weights) / total_weight + target = float(np.clip(percentile, 0.0, 100.0)) / 100.0 + return float(np.interp(target, cumulative, values, left=values[0], right=values[-1])) + + +def weighted_nanstd(values, weights): + values = np.asarray(values, dtype=float).ravel() + weights = np.asarray(weights, dtype=float).ravel() + valid = np.isfinite(values) & np.isfinite(weights) & (weights > 0) + if not np.any(valid): + return np.nan + + values = values[valid] + weights = weights[valid] + total_weight = float(np.sum(weights)) + if not np.isfinite(total_weight) or total_weight <= 0: + return np.nan + + mean = float(np.sum(weights * values) / total_weight) + variance = float(np.sum(weights * (values - mean) ** 2) / total_weight) + return float(np.sqrt(max(variance, 0.0))) + + +def sigma_clipped_weighted_median(values, weights, sigma=3.0, max_iters=3, high_only=False): + values = np.asarray(values, dtype=float).ravel() + weights = np.asarray(weights, dtype=float).ravel() + keep = np.isfinite(values) & np.isfinite(weights) & (weights > 0) + if not np.any(keep): + return np.nan, np.nan + + for _ in range(max_iters): + center = weighted_nanpercentile(values[keep], weights[keep], 50.0) + scatter = weighted_nanstd(values[keep], weights[keep]) + if not np.isfinite(center): + return np.nan, np.nan + if not np.isfinite(scatter) or scatter <= 0: + break + + if high_only: + updated_keep = keep & (values <= center + sigma * scatter) + else: + updated_keep = keep & (np.abs(values - center) <= sigma * scatter) + if np.array_equal(updated_keep, keep): + break + keep = updated_keep + + if not np.any(keep): + return np.nan, np.nan + + return weighted_nanpercentile(values[keep], weights[keep], 50.0), weighted_nanstd(values[keep], weights[keep]) + + +def psf_fwhm_from_sigma(sigma): + try: + sigma = float(sigma) + except (TypeError, ValueError): + return np.nan + + if not np.isfinite(sigma) or sigma <= 0: + return np.nan + + return float(GAUSSIAN_SIGMA_TO_FWHM * sigma) + + +def resolve_sky_annulus_geometry( + aperture_radius, + annulus_width, + psf_sigma=np.nan, + minimum_gap_pixels=SKY_ANNULUS_MIN_GAP_PIXELS, + minimum_fwhm_multiplier=SKY_ANNULUS_MIN_FWHM_MULTIPLIER, + minimum_sky_pixels=SKY_ANNULUS_MIN_EFFECTIVE_PIXELS, +): + aperture_radius = abs(float(aperture_radius)) + annulus_width = max(float(annulus_width), 0.0) + + inner_radius = aperture_radius + float(minimum_gap_pixels) + fwhm = psf_fwhm_from_sigma(psf_sigma) + if np.isfinite(fwhm): + inner_radius = max(inner_radius, float(minimum_fwhm_multiplier) * fwhm) + + outer_radius = inner_radius + annulus_width + effective_sky_pixels = np.pi * max(outer_radius ** 2 - inner_radius ** 2, 0.0) + + if minimum_sky_pixels is not None and np.isfinite(minimum_sky_pixels) and minimum_sky_pixels > 0: + minimum_outer_radius = float(np.sqrt(inner_radius ** 2 + float(minimum_sky_pixels) / np.pi)) + if minimum_outer_radius > outer_radius: + outer_radius = minimum_outer_radius + effective_sky_pixels = np.pi * max(outer_radius ** 2 - inner_radius ** 2, 0.0) + + return { + 'inner_radius': float(inner_radius), + 'outer_radius': float(outer_radius), + 'annulus_width': float(max(outer_radius - inner_radius, 0.0)), + 'effective_sky_pixels': float(effective_sky_pixels), + 'fwhm': float(fwhm) if np.isfinite(fwhm) else np.nan, + } + + +def finite_positive_or_nan(value): + try: + value = float(value) + except (TypeError, ValueError): + return np.nan + + if not np.isfinite(value) or value <= 0: + return np.nan + return value + + +def _aperture_correction_fallback_fwhm(fwhm_hint=np.nan, fallback_sigma=np.nan): + fwhm = finite_positive_or_nan(fwhm_hint) + if np.isfinite(fwhm): + return fwhm + + sigma = finite_positive_or_nan(fallback_sigma) + if np.isfinite(sigma): + return psf_fwhm_from_sigma(sigma) + + return np.nan + + +def _limited_bright_pixel_indices(search_image, bright, max_count): + bright_count = int(np.count_nonzero(bright)) + if bright_count == 0: + return np.empty(0, dtype=np.intp) + if bright_count <= max_count: + return np.flatnonzero(bright.ravel()) + + height, width = bright.shape + selected_blocks = [] + for y0 in range(0, height, APERTURE_CORRECTION_PEAK_BLOCK_SIZE): + y1 = min(y0 + APERTURE_CORRECTION_PEAK_BLOCK_SIZE, height) + for x0 in range(0, width, APERTURE_CORRECTION_PEAK_BLOCK_SIZE): + x1 = min(x0 + APERTURE_CORRECTION_PEAK_BLOCK_SIZE, width) + block_bright = bright[y0:y1, x0:x1] + block_count = int(np.count_nonzero(block_bright)) + if block_count == 0: + continue + + if block_count <= APERTURE_CORRECTION_PEAK_BLOCK_LIMIT: + local_flat = np.flatnonzero(block_bright.ravel()) + else: + block_scores = np.where(block_bright, search_image[y0:y1, x0:x1], -np.inf) + local_flat = np.argpartition( + block_scores.ravel(), + -APERTURE_CORRECTION_PEAK_BLOCK_LIMIT, + )[-APERTURE_CORRECTION_PEAK_BLOCK_LIMIT:] + local_flat = local_flat[np.isfinite(block_scores.ravel()[local_flat])] + + yy, xx = np.divmod(local_flat, x1 - x0) + selected_blocks.append((yy + y0) * width + (xx + x0)) + + if not selected_blocks: + return np.empty(0, dtype=np.intp) + + selected = np.concatenate(selected_blocks).astype(np.intp, copy=False) + if selected.size <= max_count: + return selected + + selected_values = search_image.ravel()[selected] + strongest = np.argpartition(selected_values, -max_count)[-max_count:] + return selected[strongest] + + +def detect_aperture_correction_star_candidates(data, fwhm_hint=np.nan): + image = np.asarray(data, dtype=float) + if image.ndim != 2: + return np.empty((0, 3), dtype=float) + if image.shape[0] < 3 or image.shape[1] < 3: + return np.empty((0, 3), dtype=float) + + finite = np.isfinite(image) + if not np.any(finite): + return np.empty((0, 3), dtype=float) + + background, scatter = sigma_clipped_nanmedian(image[finite], sigma=3.0, max_iters=3) + if not np.isfinite(background): + background = float(np.nanmedian(image[finite])) + if not np.isfinite(scatter) or scatter <= 0: + scatter = float(np.nanstd(image[finite])) + if not np.isfinite(scatter) or scatter <= 0: + return np.empty((0, 3), dtype=float) + + fwhm = finite_positive_or_nan(fwhm_hint) + if not np.isfinite(fwhm): + fwhm = 3.0 + fwhm = float(np.clip(fwhm, 1.0, 20.0)) + + search_image = np.where(finite, image - background, -np.inf) + threshold = APERTURE_CORRECTION_DETECTION_SIGMA * scatter + bright = finite & (search_image > threshold) + bright[0, :] = False + bright[-1, :] = False + bright[:, 0] = False + bright[:, -1] = False + + bright_flat = _limited_bright_pixel_indices( + search_image, + bright, + APERTURE_CORRECTION_PEAK_TEST_LIMIT, + ) + if bright_flat.size == 0: + return np.empty((0, 3), dtype=float) + + y, x = np.divmod(bright_flat, image.shape[1]) + flux = search_image[y, x] + order = np.argsort(flux, kind='mergesort')[::-1] + local_radius = int(np.clip(np.ceil(0.5 * fwhm), 1, 10)) + suppression_radius = max(1.0, 0.75 * fwhm) + suppression_radius_sq = suppression_radius * suppression_radius + + rows = [] + accepted_xy = [] + for idx in order: + xc = int(x[idx]) + yc = int(y[idx]) + center_flux = float(flux[idx]) + if not np.isfinite(center_flux): + continue + + y0 = yc - local_radius + y1 = yc + local_radius + 1 + x0 = xc - local_radius + x1 = xc + local_radius + 1 + if y0 < 0 or x0 < 0 or y1 > image.shape[0] or x1 > image.shape[1]: + continue + if center_flux < float(np.nanmax(search_image[y0:y1, x0:x1])): + continue + + if accepted_xy: + accepted = np.asarray(accepted_xy, dtype=float) + if np.any((accepted[:, 0] - xc) ** 2 + (accepted[:, 1] - yc) ** 2 <= suppression_radius_sq): + continue + + rows.append((float(xc), float(yc), center_flux)) + accepted_xy.append((float(xc), float(yc))) + if len(rows) >= APERTURE_CORRECTION_MAX_DETECTED_STARS: + break + + if not rows: + return np.empty((0, 3), dtype=float) + return np.asarray(rows, dtype=float) + + +def isolated_aperture_correction_candidates(candidates, image_shape, fwhm_hint=np.nan): + candidates = np.asarray(candidates, dtype=float) + if candidates.ndim != 2 or candidates.shape[1] < 2 or candidates.size == 0: + return np.empty((0, 3), dtype=float) + + fwhm = finite_positive_or_nan(fwhm_hint) + if not np.isfinite(fwhm): + fwhm = 3.0 + + height, width = image_shape[:2] + min_separation = APERTURE_CORRECTION_MIN_SEPARATION_FWHM * fwhm + border = max( + APERTURE_CORRECTION_MIN_BORDER_PIXELS, + APERTURE_CORRECTION_MIN_SEPARATION_FWHM * fwhm, + ) + + x = candidates[:, 0] + y = candidates[:, 1] + keep = ( + np.isfinite(x) + & np.isfinite(y) + & (x >= border) + & (x <= (width - 1 - border)) + & (y >= border) + & (y <= (height - 1 - border)) + ) + candidates = candidates[keep] + if candidates.shape[0] <= 1: + return candidates[:APERTURE_CORRECTION_MAX_STARS] + + x = candidates[:, 0] + y = candidates[:, 1] + distances = np.hypot(x[:, None] - x[None, :], y[:, None] - y[None, :]) + np.fill_diagonal(distances, np.inf) + nearest = np.min(distances, axis=1) + candidates = candidates[nearest >= min_separation] + if candidates.size == 0: + return np.empty((0, 3), dtype=float) + + order = np.argsort(candidates[:, 2], kind='mergesort')[::-1] + return candidates[order[:APERTURE_CORRECTION_MAX_STARS]] + + +def estimate_isolated_field_star_psfs(data, fwhm_hint=np.nan): + image = np.asarray(data, dtype=float) + if image.ndim != 2: + return np.empty((0, 7), dtype=float) + + candidates = detect_aperture_correction_star_candidates(image, fwhm_hint=fwhm_hint) + candidates = isolated_aperture_correction_candidates(candidates, image.shape, fwhm_hint=fwhm_hint) + if candidates.size == 0: + return np.empty((0, 7), dtype=float) + + fwhm = finite_positive_or_nan(fwhm_hint) + if not np.isfinite(fwhm): + fwhm = 3.0 + box = int(np.clip(np.ceil(3.0 * fwhm), 6, 30)) + + rows = [] + for x, y, _ in candidates: + try: + xv, yv = mesh_box([x, y], box, maxx=image.shape[1], maxy=image.shape[0]) + subarray = image[yv, xv] + except Exception: + continue + + if subarray.size == 0: + continue + + row = _fit_centroid_moments(subarray, xv, yv, [x, y], box) + if not np.all(np.isfinite(row[:5])): + continue + if not _has_usable_centroid_signal(subarray, row[2], min_snr=5.0): + continue + + solved_fwhm = psf_fwhm_from_sigma(0.5 * (row[3] + row[4])) + if not np.isfinite(solved_fwhm): + continue + if np.hypot(row[0] - x, row[1] - y) > max(2.0, 0.75 * solved_fwhm): + continue + + rows.append(row) + + if not rows: + return np.empty((0, 7), dtype=float) + return np.asarray(rows, dtype=float) + + +def image_fwhm_from_field_star_psfs(field_star_psfs, fallback_fwhm=np.nan): + rows = np.asarray(field_star_psfs, dtype=float) + if rows.ndim != 2 or rows.shape[1] < 5 or rows.size == 0: + return finite_positive_or_nan(fallback_fwhm) + + sigmas = 0.5 * (rows[:, 3] + rows[:, 4]) + fwhm_values = GAUSSIAN_SIGMA_TO_FWHM * sigmas + fwhm_values[~np.isfinite(fwhm_values) | (fwhm_values <= 0)] = np.nan + center, _ = sigma_clipped_nanmedian(fwhm_values, sigma=3.0, max_iters=3) + if np.isfinite(center) and center > 0: + return float(center) + + return finite_positive_or_nan(fallback_fwhm) + + +def estimate_image_fwhm_from_isolated_stars(data, fwhm_hint=np.nan, fallback_sigma=np.nan): + fallback_fwhm = _aperture_correction_fallback_fwhm(fwhm_hint, fallback_sigma) + field_star_psfs = estimate_isolated_field_star_psfs(data, fwhm_hint=fallback_fwhm) + return image_fwhm_from_field_star_psfs(field_star_psfs, fallback_fwhm=fallback_fwhm) + + +def _aperture_correction_sky_background(data, xc, yc, reference_radius, image_fwhm, fast_mode=False): + sigma_hint = image_fwhm / GAUSSIAN_SIGMA_TO_FWHM if np.isfinite(image_fwhm) and image_fwhm > 0 else np.nan + sky_geometry = resolve_sky_annulus_geometry( + reference_radius, + max(float(image_fwhm), 3.0) if np.isfinite(image_fwhm) else 5.0, + psf_sigma=sigma_hint, + ) + + try: + annulus = CircularAnnulus( + positions=[(xc, yc)], + r_in=sky_geometry['inner_radius'], + r_out=sky_geometry['outer_radius'], + ) + mask_method = 'center' if fast_mode else 'exact' + annulus_mask = annulus.to_mask(method=mask_method)[0] + annulus_cutout = annulus_mask.cutout(data, fill_value=np.nan) + except Exception: + return np.nan + + if annulus_cutout is None: + return np.nan + + annulus_cutout = np.asarray(annulus_cutout, dtype=float) + annulus_weights = np.asarray(annulus_mask.data, dtype=float) + valid_mask = np.isfinite(annulus_cutout) & np.isfinite(annulus_weights) & (annulus_weights > 0) + if not np.any(valid_mask): + return np.nan + + annulus_pixels = annulus_cutout[valid_mask] + annulus_pixel_weights = annulus_weights[valid_mask] + cutoff = weighted_nanpercentile(annulus_pixels, annulus_pixel_weights, 99) + if not np.isfinite(cutoff): + return np.nan + + clipped_keep = annulus_pixels <= cutoff + if not np.any(clipped_keep): + return np.nan + + sky_median, _ = sigma_clipped_weighted_median( + annulus_pixels[clipped_keep], + annulus_pixel_weights[clipped_keep], + sigma=SKY_BACKGROUND_SIGMA_CLIP, + max_iters=SKY_BACKGROUND_SIGMA_CLIP_MAX_ITERS, + high_only=True, + ) + return sky_median + + +def _background_subtracted_aperture_sum(data, xc, yc, radius, background, fast_mode=False): + radius = finite_positive_or_nan(radius) + if not np.isfinite(radius) or not np.isfinite(background): + return np.nan + + try: + aperture = CircularAperture(positions=[(xc, yc)], r=radius) + mask_method = 'center' if fast_mode else 'exact' + mask = aperture.to_mask(method=mask_method)[0] + data_cutout = mask.cutout(data) + except Exception: + return np.nan + + if data_cutout is None: + return np.nan + + weights = np.asarray(mask.data, dtype=float) + values = np.asarray(data_cutout, dtype=float) + valid = np.isfinite(weights) & np.isfinite(values) & (weights > 0) + if not np.any(valid): + return np.nan + + return float(np.sum(weights[valid] * (values[valid] - background))) + + +def build_aperture_correction_profile(data, aperture_radii, fwhm_hint=np.nan, fast_mode=False, + field_star_psfs=None): + aperture_radii = np.asarray(aperture_radii, dtype=float).reshape(-1) + correction_factors = np.ones(aperture_radii.shape, dtype=float) + fallback_fwhm = finite_positive_or_nan(fwhm_hint) + + if field_star_psfs is None: + field_star_psfs = estimate_isolated_field_star_psfs(data, fwhm_hint=fallback_fwhm) + else: + field_star_psfs = np.asarray(field_star_psfs, dtype=float) + + image_fwhm = image_fwhm_from_field_star_psfs(field_star_psfs, fallback_fwhm=fallback_fwhm) + profile = { + 'applied': False, + 'image_fwhm': image_fwhm, + 'star_count': int(field_star_psfs.shape[0]) if field_star_psfs.ndim == 2 else 0, + 'aperture_radii': aperture_radii, + 'correction_factors': correction_factors, + 'curve_radii': np.array([], dtype=float), + 'enclosed_fraction': np.array([], dtype=float), + 'note': 'Aperture correction skipped; no aperture radii were provided.', + } + + valid_radius_mask = np.isfinite(aperture_radii) & (aperture_radii > 0) + if not np.any(valid_radius_mask): + return profile + + if not np.isfinite(image_fwhm) or image_fwhm <= 0: + profile['note'] = 'Aperture correction skipped; image FWHM could not be estimated.' + return profile + + if profile['star_count'] < APERTURE_CORRECTION_MIN_STARS: + profile['note'] = ( + "Aperture correction skipped; fewer than " + f"{APERTURE_CORRECTION_MIN_STARS} isolated field stars were available." + ) + return profile + + reference_radius = APERTURE_MAX_FWHM_MULTIPLIER * image_fwhm + measurement_radii = np.unique(np.concatenate([aperture_radii[valid_radius_mask], [reference_radius]])) + measurement_radii = measurement_radii[np.isfinite(measurement_radii) & (measurement_radii > 0)] + if measurement_radii.size == 0: + return profile + + fractions_by_radius = {float(radius): [] for radius in measurement_radii} + for row in field_star_psfs: + xc, yc = float(row[0]), float(row[1]) + background = _aperture_correction_sky_background( + data, + xc, + yc, + reference_radius, + image_fwhm, + fast_mode=fast_mode, + ) + reference_flux = _background_subtracted_aperture_sum( + data, + xc, + yc, + reference_radius, + background, + fast_mode=fast_mode, + ) + if not np.isfinite(reference_flux) or reference_flux <= 0: + continue + + for radius in measurement_radii: + flux = _background_subtracted_aperture_sum( + data, + xc, + yc, + radius, + background, + fast_mode=fast_mode, + ) + fraction = flux / reference_flux if np.isfinite(flux) else np.nan + if np.isfinite(fraction) and fraction > 0: + fractions_by_radius[float(radius)].append(float(fraction)) + + curve_radii = [] + enclosed_fraction = [] + for radius in measurement_radii: + fractions = np.asarray(fractions_by_radius[float(radius)], dtype=float) + if fractions.size == 0: + continue + center, _ = sigma_clipped_nanmedian(fractions, sigma=3.0, max_iters=3) + if np.isfinite(center) and center > 0: + curve_radii.append(float(radius)) + enclosed_fraction.append(float(center)) + + if not curve_radii: + profile['note'] = 'Aperture correction skipped; isolated-star curve of growth could not be measured.' + return profile + + curve_radii = np.asarray(curve_radii, dtype=float) + enclosed_fraction = np.asarray(enclosed_fraction, dtype=float) + order = np.argsort(curve_radii, kind='mergesort') + curve_radii = curve_radii[order] + enclosed_fraction = enclosed_fraction[order] + enclosed_fraction = np.clip(enclosed_fraction, 1.0 / APERTURE_CORRECTION_MAX_FACTOR, 1.0) + enclosed_fraction = np.maximum.accumulate(enclosed_fraction) + enclosed_fraction = np.minimum(enclosed_fraction, 1.0) + + interpolated_fraction = np.interp( + aperture_radii[valid_radius_mask], + curve_radii, + enclosed_fraction, + left=enclosed_fraction[0], + right=1.0, + ) + interpolated_fraction = np.clip(interpolated_fraction, 1.0 / APERTURE_CORRECTION_MAX_FACTOR, 1.0) + correction_factors[valid_radius_mask] = np.clip( + 1.0 / interpolated_fraction, + 1.0, + APERTURE_CORRECTION_MAX_FACTOR, + ) + + profile.update({ + 'applied': True, + 'correction_factors': correction_factors, + 'curve_radii': curve_radii, + 'enclosed_fraction': enclosed_fraction, + 'reference_radius': float(reference_radius), + 'note': ( + "Applied aperture correction from " + f"{profile['star_count']} isolated field star(s); image FWHM={image_fwhm:.2f}px." + ), + }) + return profile + + +# Method calculates the flux of the star (uses the skybg_phot method to do background sub) +def aperPhot(data, starIndex, xc, yc, r=5, dr=5, fast_mode=False, sigma_hint=np.nan): + stage_start = perf_counter() + try: + # Check for invalid coordinates + if np.isnan(xc) or np.isnan(yc): + return 0, 0 + + # Calculate background if dr > 0 + if dr > 0: + sky_geometry = resolve_sky_annulus_geometry(r, dr, psf_sigma=sigma_hint) + bgflux, sigmabg, Nbg = skybg_phot( + data, + starIndex, + xc, + yc, + sky_geometry['inner_radius'], + sky_geometry['annulus_width'], + fast_mode=fast_mode, + ) + if not np.isfinite(bgflux): + return np.nan, bgflux + else: + bgflux, sigmabg, Nbg = 0, 0, 0 + + # Create aperture and mask + aperture = CircularAperture(positions=[(xc, yc)], r=r) + mask_method = 'center' if fast_mode else 'exact' + mask = aperture.to_mask(method=mask_method)[0] + data_cutout = mask.cutout(data) + + # Check if aperture is valid + if data_cutout is None: + # Aperture is partially or fully outside the image + return 0, bgflux # Return zero flux but valid background + + # Calculate and return aperture sum + aperture_sum = (mask.data * (data_cutout - bgflux)).sum() + return aperture_sum, bgflux + finally: + _record_photometry_stage_timing('aperPhot', perf_counter() - stage_start) + + +def skybg_phot(data, starIndex, xc, yc, r=10, dr=5, ptol=99, debug=False, fast_mode=False): + # The sky annulus uses an inner radius r and an outer radius r + dr. + # Callers are responsible for choosing r and dr from the aperture radius and PSF size. + annulus = CircularAnnulus(positions=[(xc, yc)], r_in=float(r), r_out=float(r + dr)) + mask_method = 'center' if fast_mode else 'exact' + annulus_mask = annulus.to_mask(method=mask_method)[0] + annulus_cutout = annulus_mask.cutout(data, fill_value=np.nan) + + if annulus_cutout is None: + plateStatus.skyBackgroundWarning(starIndex, xc, yc) + log.debug(f"Warning: empty sky background annulus for {xc:.1f}, {yc:.1f}." + f"\nCheck if star is present or close to border.") + return np.nan, np.nan, 0 + + annulus_cutout = np.asarray(annulus_cutout, dtype=float) + annulus_weights = np.asarray(annulus_mask.data, dtype=float) + valid_mask = np.isfinite(annulus_cutout) & np.isfinite(annulus_weights) & (annulus_weights > 0) + if not np.any(valid_mask): + plateStatus.skyBackgroundWarning(starIndex, xc, yc) + log.debug(f"Warning: no valid sky background pixels for {xc:.1f}, {yc:.1f}." + f"\nCheck if star is present or close to border.") + return np.nan, np.nan, 0 + + annulus_pixels = annulus_cutout[valid_mask] + annulus_pixel_weights = annulus_weights[valid_mask] + + try: + cutoff = weighted_nanpercentile(annulus_pixels, annulus_pixel_weights, ptol) + except (IndexError, ValueError): + plateStatus.skyBackgroundWarning(starIndex, xc, yc) + log.debug(f"Warning: IndexError, problem computing sky bg for {xc:.1f}, {yc:.1f}." + f"\nCheck if star is present or close to border.") + return np.nan, np.nan, 0 + + if not np.isfinite(cutoff): + plateStatus.skyBackgroundWarning(starIndex, xc, yc) + log.debug(f"Warning: invalid cutoff while computing sky bg for {xc:.1f}, {yc:.1f}.") + return np.nan, np.nan, 0 + + clipped_keep = annulus_pixels <= cutoff + clipped_pixels = annulus_pixels[clipped_keep] + clipped_weights = annulus_pixel_weights[clipped_keep] + if clipped_pixels.size == 0: + plateStatus.skyBackgroundWarning(starIndex, xc, yc) + log.debug(f"Warning: percentile clipping removed all sky background pixels for {xc:.1f}, {yc:.1f}.") + return np.nan, np.nan, 0 + + dat = np.full_like(annulus_cutout, np.nan, dtype=float) + dat[valid_mask] = annulus_cutout[valid_mask] + dat[valid_mask & (annulus_cutout > cutoff)] = np.nan + + if debug: + minb = float(np.nanmin(annulus_pixels)) + maxb = float(np.nanmean(annulus_pixels) + 3 * np.nanstd(annulus_pixels)) + bgsky = np.full_like(annulus_cutout, np.nan, dtype=float) + bgsky[valid_mask] = annulus_cutout[valid_mask] + cmed, _ = sigma_clipped_weighted_median( + clipped_pixels, + clipped_weights, + sigma=SKY_BACKGROUND_SIGMA_CLIP, + max_iters=SKY_BACKGROUND_SIGMA_CLIP_MAX_ITERS, + high_only=True, + ) + amed, _ = sigma_clipped_weighted_median( + annulus_pixels, + annulus_pixel_weights, + sigma=SKY_BACKGROUND_SIGMA_CLIP, + max_iters=SKY_BACKGROUND_SIGMA_CLIP_MAX_ITERS, + high_only=True, + ) + + fig, ax = plt.subplots(2, 2, figsize=(9, 9)) + im = ax[0, 0].imshow(annulus_cutout, vmin=minb, vmax=maxb, cmap='inferno') + ax[0, 0].set_title("Original Data") + from mpl_toolkits.axes_grid1 import make_axes_locatable + divider = make_axes_locatable(ax[0, 0]) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + + ax[1, 0].hist(annulus_pixels, label=f'Sky Annulus ({np.nanmedian(annulus_pixels):.1f}, {amed:.1f})', + alpha=0.5, bins=np.arange(minb, maxb)) + ax[1, 0].hist(clipped_pixels, label=f'Clipped ({np.nanmedian(clipped_pixels):.1f}, {cmed:.1f})', alpha=0.5, + bins=np.arange(minb, maxb)) + ax[1, 0].legend(loc='best') + ax[1, 0].set_title("Sky Background") + ax[1, 0].set_xlabel("Pixel Value") + + ax[1, 1].imshow(dat, vmin=minb, vmax=maxb, cmap='inferno') + ax[1, 1].set_title("Clipped Sky Background") + + ax[0, 1].imshow(bgsky, vmin=minb, vmax=maxb, cmap='inferno') + ax[0, 1].set_title("Sky Annulus") + plt.tight_layout() + plt.show() + sky_median, sky_sigma = sigma_clipped_weighted_median( + clipped_pixels, + clipped_weights, + sigma=SKY_BACKGROUND_SIGMA_CLIP, + max_iters=SKY_BACKGROUND_SIGMA_CLIP_MAX_ITERS, + high_only=True, + ) + return sky_median, sky_sigma, float(np.sum(annulus_pixel_weights)) + +def process_dark_frames(dark_files): + """Process dark frames and return the master dark.""" + if not dark_files: + return None + # Dark files whose median is much higher than the overall dark files median will be filtered + # e.g. to discard saturated dark files that may negatively affect the master dark used to calibrate the science frames + # First pass: collect all dark frame medians + darks_medians = [(dark_file, np.nanmedian(fits.getdata(dark_file))) for dark_file in dark_files] + + d_median = np.median([median for _, median in darks_medians]) + threshold = 1.7 # 70% higher than overall median + + # Second pass: collect valid dark frames + darks_img_list = [] + for dark_file, dark_median in darks_medians: + median_ratio = dark_median / d_median + if median_ratio > threshold: + log_info( + f"\nWarning: Skipping suspicious dark frame {dark_file}: " + f"median/overall_median = {median_ratio:.2f}\n", + warn=True + ) + continue + dark_data = fits.getdata(dark_file) + darks_img_list.append(dark_data) + + return np.median(darks_img_list, axis=0) if darks_img_list else None + +def process_bias_frames(bias_files): + """Process bias frames and return the master bias.""" + if not bias_files: + return None + + biases_img_list = [fits.getdata(bias_file) for bias_file in bias_files] + return np.median(biases_img_list, axis=0) if biases_img_list else None + +def process_flat_frames(flat_files, master_bias=None): + """Process flat frames and return the normalized master flat.""" + if not flat_files: + return None + + flats_img_list = [fits.getdata(flat_file) for flat_file in flat_files] + master_flat = np.median(flats_img_list, axis=0) + # Bias subtract after creating master flat + if master_bias is not None: + master_flat = master_flat - master_bias + # Normalize + medi = np.median(master_flat) + return master_flat / medi + +def convert_jd_to_bjd(non_bjd, p_dict, info_dict): + global _BJD_FALLBACK_WARNING_LOGGED + + try: + goodTimes = JDUTC_to_BJDTDB(non_bjd, ra=p_dict['ra'], dec=p_dict['dec'], lat=info_dict['lat'], + longi=info_dict['long'], alt=info_dict['elev'])[0] + except Exception as exc: + if not _BJD_FALLBACK_WARNING_LOGGED: + _BJD_FALLBACK_WARNING_LOGGED = True + try: + log.warning( + "barycorrpy JDUTC_to_BJDTDB conversion failed; falling back to astropy light-travel-time " + "conversion for this run.", + exc_info=True, + ) + except Exception: + traceback.print_exception(type(exc), exc, exc.__traceback__, file=sys.stdout) + targetloc = SkyCoord(p_dict['ra'], p_dict['dec'], unit=(u.deg, u.deg), frame='icrs') + obsloc = EarthLocation(lat=info_dict['lat'], lon=info_dict['long'], height=info_dict['elev']) + timesToConvert = Time(non_bjd, format='jd', scale='utc', location=obsloc) + ltt_bary = timesToConvert.light_travel_time(targetloc) + time_barycentre = timesToConvert.tdb + ltt_bary + goodTimes = time_barycentre.value + + return goodTimes + + +def calculate_variablility(fit_lc_ref, fit_lc_best): + info_ref = None + + mask_oot_ref = (fit_lc_ref.transit == 1) + mask_oot_best = (fit_lc_best.transit == 1) + + intx_times = np.intersect1d(fit_lc_best.jd_times[mask_oot_best], fit_lc_ref.jd_times[mask_oot_ref]) + + if intx_times.any(): + mask_ref = np.isin(fit_lc_ref.jd_times, intx_times) + mask_best = np.isin(fit_lc_best.jd_times, intx_times) + + norm_flux_ref = (fit_lc_ref.data / np.nanmedian(fit_lc_ref.data[mask_ref]))[mask_ref] + norm_flux_best = (fit_lc_best.data / np.nanmedian(fit_lc_best.data[mask_best]))[mask_best] + + info_ref = { + 'fit_lc': fit_lc_ref, + 'mask_ref': mask_ref, + 'res': norm_flux_best - norm_flux_ref, + } + + return info_ref + + +def choose_comp_star_variability(fit_lc_refs, fit_lc_best, ref_comp, comp_stars, vsp_comp_stars, save): + colors = ["firebrick", "darkorange", "olivedrab", "lightseagreen", "steelblue", "rebeccapurple", "mediumvioletred"] + markers = ['.', 'v', 's', 'D', '^'] + k = 0 + + labels = {tuple(value['pos']): key for key, value in vsp_comp_stars.items()} + + for i, ckey in enumerate(fit_lc_refs.keys()): + if i >= len(colors): + i = 0 + if k >= len(markers): + k = 0 + ref_comp[ckey] = calculate_variablility(fit_lc_refs[ckey]['myfit'], fit_lc_best) + + if ref_comp[ckey]: + plt.errorbar(ref_comp[ckey]['fit_lc'].jd_times[ref_comp[ckey]['mask_ref']], ref_comp[ckey]['res'], + fmt=markers[k], color=colors[i], label=f"{labels[tuple(fit_lc_refs[ckey]['pos'])]}") + k += 1 + + plot_variable_residuals(save) + + std_devs = {key: np.std(value['res']) for key, value in ref_comp.items() if value} + if not std_devs: + raise RuntimeError("No usable comparison-star residuals were available for stellar variability calibration.") + min_std_dev = min(std_devs, key=lambda y: abs(std_devs[y])) + + return comp_stars[min_std_dev] + + +def stellar_variability_label(comp_label, comp_star): + if comp_star.get('is_aavso_vsp', True): + return comp_label + comp_ra = _finite_float(comp_star.get('ra')) + comp_dec = _finite_float(comp_star.get('dec')) + if comp_ra is not None and comp_dec is not None: + return f"RA={comp_ra:.7f} Dec={comp_dec:.7f}" + return comp_label + + +def annotate_differential_magnitude_raw_photometry(lc_fit, target_flux, reference_flux, + target_flux_error=None, + reference_flux_error=None): + """Retain the unnormalized flux pair used by differential-magnitude outputs.""" + if lc_fit is None: + return False + + fit_shape = np.asarray(getattr(lc_fit, 'data', []), dtype=float).shape + target_flux = np.asarray(target_flux if target_flux is not None else [], dtype=float) + reference_flux = np.asarray(reference_flux if reference_flux is not None else [], dtype=float) + if target_flux.shape != fit_shape or reference_flux.shape != fit_shape: + return False + + def aligned_error(values): + if values is None: + return np.full(fit_shape, np.nan, dtype=float) + array = np.asarray(values, dtype=float) + if array.shape != fit_shape: + return np.full(fit_shape, np.nan, dtype=float) + return array + + lc_fit.differential_magnitude_target_flux = target_flux.copy() + lc_fit.differential_magnitude_reference_flux = reference_flux.copy() + lc_fit.differential_magnitude_target_flux_error = aligned_error(target_flux_error).copy() + lc_fit.differential_magnitude_reference_flux_error = aligned_error(reference_flux_error).copy() + return True + + +def annotate_stellar_variability_raw_photometry(lc_fit, target_flux, comp_flux, + target_flux_error=None, comp_flux_error=None): + if lc_fit is None: + return lc_fit + + fit_shape = np.asarray(getattr(lc_fit, 'data', []), dtype=float).shape + target_flux = np.asarray(target_flux if target_flux is not None else [], dtype=float) + comp_flux = np.asarray(comp_flux if comp_flux is not None else [], dtype=float) + if target_flux.shape != fit_shape or comp_flux.shape != fit_shape: + return lc_fit + + def aligned_error(values): + if values is None: + return np.full(fit_shape, np.nan, dtype=float) + array = np.asarray(values, dtype=float) + if array.shape != fit_shape: + return np.full(fit_shape, np.nan, dtype=float) + return array + + lc_fit.stellar_variability_target_flux = target_flux.copy() + lc_fit.stellar_variability_comp_flux = comp_flux.copy() + lc_fit.stellar_variability_target_flux_error = aligned_error(target_flux_error).copy() + lc_fit.stellar_variability_comp_flux_error = aligned_error(comp_flux_error).copy() + return lc_fit + + +def stellar_variability_raw_photometry(lc_fit): + fit_shape = np.asarray(getattr(lc_fit, 'data', []), dtype=float).shape + target_flux = np.asarray( + getattr(lc_fit, 'stellar_variability_target_flux', []), + dtype=float, + ) + comp_flux = np.asarray( + getattr(lc_fit, 'stellar_variability_comp_flux', []), + dtype=float, + ) + if target_flux.shape != fit_shape or comp_flux.shape != fit_shape: + raise RuntimeError( + "Raw target and comparison fluxes are unavailable; an absolute differential magnitude " + "cannot be recovered from a normalized light curve." + ) + + target_flux_error = np.asarray( + getattr(lc_fit, 'stellar_variability_target_flux_error', []), + dtype=float, + ) + comp_flux_error = np.asarray( + getattr(lc_fit, 'stellar_variability_comp_flux_error', []), + dtype=float, + ) + if target_flux_error.shape != fit_shape: + target_flux_error = np.full(fit_shape, np.nan, dtype=float) + if comp_flux_error.shape != fit_shape: + comp_flux_error = np.full(fit_shape, np.nan, dtype=float) + return target_flux, comp_flux, target_flux_error, comp_flux_error + + +def build_stellar_variability_params_from_fit(lc_fit, comp_star, comp_pos, comp_label, save, s_name, + observed_filter=None, observation_date=None): + # Differential photometry is independent of catalogue calibration. Emit + # it before validating the comparison magnitude so a failed apparent- + # magnitude attempt can never suppress the raw stellar-variability output. + save_stellar_variability_differential_products( + lc_fit, + save, + s_name, + observation_date=observation_date, + observed_filter=observed_filter, + ) + comp_mag = _finite_float(comp_star.get('mag')) + comp_mag_error = normalized_magnitude_error(comp_star.get('error')) + derived_catalog_reference = bool(comp_star.get('derived_catalog_reference', False)) + allow_high_error_catalog_reference = bool(comp_star.get('allow_high_error_catalog_reference', False)) + allow_relaxed_bv_error = ( + bool(comp_star.get('uses_relaxed_bv_error_limit', False)) + and str(comp_star.get('mag_band') or '').strip().upper() in {'B', 'V'} + and comp_mag_error is not None + and comp_mag_error <= CATALOG_BV_REFERENCE_MAGNITUDE_ERROR_FALLBACK_MAX + ) + if ( + comp_mag is None + or comp_mag_error is None + or ( + comp_mag_error > CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX + and not derived_catalog_reference + and not allow_high_error_catalog_reference + and not allow_relaxed_bv_error + ) + or not is_usable_apparent_magnitude(comp_mag) + ): + raise RuntimeError("Comparison-star magnitude or magnitude uncertainty is unavailable.") + observed_filter = observed_filter or comp_star.get('observed_filter') + catalog_mag_band = comp_star.get('mag_band', 'V') + if catalog_band_priority(catalog_mag_band, observed_filter) != 0: + raise RuntimeError( + "Comparison-star catalog magnitude band " + f"{catalog_mag_band!r} does not match observed filter " + f"{observed_filter!r}; cross-band absolute calibration is not permitted." + ) + measurement_mag_band = reported_stellar_variability_band( + observed_filter, + fallback_band=catalog_mag_band, + ) + + fit_data = np.asarray(getattr(lc_fit, 'data', []), dtype=float) + fit_airmass = np.asarray(getattr(lc_fit, 'airmass', np.ones_like(fit_data)), dtype=float) + # Public magnitude products are labelled BJD_TDB; ``time`` is the + # barycentric series and ``jd_times`` retains the original FITS JD/UTC. + fit_times = np.asarray(getattr(lc_fit, 'time', getattr(lc_fit, 'jd_times', [])), dtype=float) + transit_model = np.asarray(getattr(lc_fit, 'transit', np.ones_like(fit_data)), dtype=float) + + if not (fit_data.shape == fit_airmass.shape == fit_times.shape): + raise RuntimeError("Lightcurve arrays have inconsistent shapes for stellar variability output.") + + target_flux, comp_flux, target_flux_error, comp_flux_error = stellar_variability_raw_photometry(lc_fit) + + if transit_model.shape == fit_data.shape: + mask_ref = transit_model == 1 + else: + mask_ref = np.ones_like(fit_data, dtype=bool) + + if np.count_nonzero(mask_ref) == 0: + mask_ref = np.isfinite(fit_data) + + target_flux = target_flux[mask_ref] + comp_flux = comp_flux[mask_ref] + target_flux_error = target_flux_error[mask_ref] + comp_flux_error = comp_flux_error[mask_ref] + selected_times = fit_times[mask_ref] + selected_airmass = fit_airmass[mask_ref] + + with np.errstate(divide='ignore', invalid='ignore'): + raw_ratio = np.divide(target_flux, comp_flux) + # Do not apply the transit fit's airmass trend to stellar-variability + # measurements. A real stellar change that is correlated with time is + # also correlated with airmass during a normal observing sequence, so the + # detrending model could suppress the astrophysical signal. + calibrated_ratio = raw_ratio + with np.errstate(divide='ignore', invalid='ignore'): + differential_mag = -2.5 * np.log10(calibrated_ratio) + target_mag = comp_mag + differential_mag + magnitude_factor = 2.5 / np.log(10.0) + explicit_flux_error = magnitude_factor * np.sqrt( + (target_flux_error / target_flux) ** 2 + + (comp_flux_error / comp_flux) ** 2 + ) + + # The fitted relative-flux uncertainty is still a valid fractional-ratio + # uncertainty after normalization, so use it only when separate stellar + # flux errors were not retained by an older caller. + fit_data_error = np.asarray(getattr(lc_fit, 'dataerr', np.full(fit_data.shape, np.nan)), dtype=float) + if fit_data_error.shape == fit_data.shape: + with np.errstate(divide='ignore', invalid='ignore'): + fallback_flux_error = magnitude_factor * np.abs( + fit_data_error[mask_ref] / fit_data[mask_ref] + ) + else: + fallback_flux_error = np.full(target_mag.shape, np.nan, dtype=float) + flux_error = np.where( + np.isfinite(explicit_flux_error) & (explicit_flux_error >= 0), + explicit_flux_error, + fallback_flux_error, + ) + target_mag_error = np.hypot(comp_mag_error, flux_error) + + valid = ( + np.isfinite(selected_times) + & np.isfinite(selected_airmass) + & np.isfinite(target_mag) + & np.isfinite(target_mag_error) + & np.isfinite(calibrated_ratio) + & (calibrated_ratio > 0) + & (target_mag <= MAX_APPARENT_MAGNITUDE) + & (target_mag_error <= MAX_APPARENT_MAGNITUDE) + ) + if np.count_nonzero(valid) == 0: + raise RuntimeError("No finite stellar variability magnitude points were produced.") + + display_label = stellar_variability_label(comp_label, comp_star) + vsp_params = [] + for time_value, airmass_value, mag_value, mag_error_value, differential_value, differential_error in zip( + selected_times[valid], + selected_airmass[valid], + target_mag[valid], + target_mag_error[valid], + differential_mag[valid], + flux_error[valid], + ): + vsp_params.append({ + 'time': time_value, + 'airmass': airmass_value, + 'mag': mag_value, + 'mag_err': mag_error_value, + 'differential_mag': differential_value, + 'differential_mag_err': differential_error, + 'cname': display_label, + 'cmag': comp_mag, + 'cmag_err': comp_mag_error, + 'pos': comp_pos, + 'comp_ra': comp_star.get('ra'), + 'comp_dec': comp_star.get('dec'), + 'catalog_ra': comp_star.get('catalog_ra'), + 'catalog_dec': comp_star.get('catalog_dec'), + 'catalog_source': comp_star.get('catalog_source', 'AAVSO VSP'), + 'is_aavso_vsp': bool(comp_star.get('is_aavso_vsp', True)), + 'mag_band': measurement_mag_band, + 'catalog_mag_band': catalog_mag_band, + 'observed_filter': observed_filter, + 'source_id': comp_star.get('source_id'), + 'catalog_id': comp_star.get('id'), + 'separation_arcsec': comp_star.get('separation_arcsec'), + 'derived_catalog_reference': derived_catalog_reference, + 'derived_reference_anchor_count': comp_star.get('derived_reference_anchor_count'), + 'derived_reference_anchor_labels': comp_star.get('derived_reference_anchor_labels'), + 'allow_high_error_catalog_reference': allow_high_error_catalog_reference, + }) + + try: + lc_fit.stellar_variability_params = vsp_params + lc_fit.stellar_variability_target_name = s_name + lc_fit.stellar_variability_reference_label = display_label + except Exception: + pass + + plot_stellar_variability(vsp_params, save, s_name, display_label) + return vsp_params + + +def stellar_variability_reference_series(lc_fit): + fit_data = np.asarray(getattr(lc_fit, 'data', []), dtype=float) + fit_times = np.asarray(getattr(lc_fit, 'time', getattr(lc_fit, 'jd_times', [])), dtype=float) + transit_model = np.asarray(getattr(lc_fit, 'transit', np.ones_like(fit_data)), dtype=float) + + if fit_data.shape != fit_times.shape: + return np.array([], dtype=float), np.array([], dtype=float) + + try: + target_flux, comp_flux, _, _ = stellar_variability_raw_photometry(lc_fit) + except RuntimeError: + return np.array([], dtype=float), np.array([], dtype=float) + with np.errstate(divide='ignore', invalid='ignore'): + reference_curve = np.divide(target_flux, comp_flux) + + valid = ( + np.isfinite(fit_times) + & np.isfinite(reference_curve) + & (reference_curve > 0) + ) + if transit_model.shape == fit_data.shape: + valid &= transit_model == 1 + + return fit_times[valid], reference_curve[valid] + + +def aligned_reference_curve_ratio(selected_fit, anchor_fit): + selected_times, selected_curve = stellar_variability_reference_series(selected_fit) + anchor_times, anchor_curve = stellar_variability_reference_series(anchor_fit) + if selected_times.size == 0 or anchor_times.size == 0: + return np.array([], dtype=float) + + selected_keys = np.round(selected_times.astype(float), 8) + anchor_keys = np.round(anchor_times.astype(float), 8) + _, selected_idx, anchor_idx = np.intersect1d( + selected_keys, + anchor_keys, + return_indices=True, + ) + if selected_idx.size == 0: + return np.array([], dtype=float) + + with np.errstate(divide='ignore', invalid='ignore'): + selected_to_anchor_flux_ratio = np.divide(anchor_curve[anchor_idx], selected_curve[selected_idx]) + + return selected_to_anchor_flux_ratio[ + np.isfinite(selected_to_anchor_flux_ratio) + & (selected_to_anchor_flux_ratio > 0) + ] + + +def build_direct_selected_catalog_candidate( + comp_stars, + comp_ra_dec, + field_catalog, + best_comp, + observed_filter=None, + match_radius_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): + if best_comp is None or best_comp < 0 or best_comp >= len(comp_stars): + return None + if field_catalog is None or not comp_ra_dec or best_comp >= len(comp_ra_dec): + return None + + comp_ra = _finite_float(comp_ra_dec[best_comp][0]) + comp_dec = _finite_float(comp_ra_dec[best_comp][1]) + if comp_ra is None or comp_dec is None: + return None + + match = nextastro_photometry_catalog_match( + field_catalog, + comp_ra, + comp_dec, + observed_filter, + max_separation_arcsec=match_radius_arcsec, + max_magnitude_error=MAX_APPARENT_MAGNITUDE, + ) + effective_match_radius = _finite_float(match_radius_arcsec) + if ( + match is None + and effective_match_radius is not None + and effective_match_radius > NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC + ): + try: + match = nextastro_photometry_for_coordinate( + comp_ra, + comp_dec, + observed_filter, + radius_arcsec=effective_match_radius, + ) + except Exception as exc: + log_info( + "Warning: scale-aware direct catalog lookup for the selected comparison star " + f"failed ({describe_retry_exception(exc)}).", + warn=True, + ) + if match is None: + return None + if catalog_band_priority(match.get('mag_band'), observed_filter) != 0: + return None + + match.update({ + 'ra': comp_ra, + 'dec': comp_dec, + 'pos': list(comp_stars[best_comp]), + 'catalog_source': 'NextAstro photometry catalog', + 'is_aavso_vsp': False, + 'observed_filter': observed_filter, + 'derived_catalog_reference': False, + 'allow_high_error_catalog_reference': True, + }) + label = unique_nextastro_calibration_label({}, match) + return { + 'label': label, + 'star': match, + 'source': 'direct_catalog', + 'error': match.get('error'), + } + + +def combine_catalog_reference_estimates(derived_estimates, selected_pos, observed_filter, source_label, + mag_band='V', label_prefix='Derived Comp', + selected_ra=None, selected_dec=None): + if not derived_estimates: + return None, None + + mags = np.asarray([estimate['mag'] for estimate in derived_estimates], dtype=float) + errors = np.asarray([estimate['error'] for estimate in derived_estimates], dtype=float) + weights = np.divide( + 1.0, + errors ** 2, + out=np.zeros_like(errors, dtype=float), + where=np.isfinite(errors) & (errors > 0), + ) + if not np.any(weights > 0): + return None, None + + combined_mag = float(np.average(mags, weights=weights)) + formal_error = float((1.0 / np.sum(weights)) ** 0.5) + if len(derived_estimates) > 1: + anchor_scatter = float(np.sqrt(np.average((mags - combined_mag) ** 2, weights=weights))) + combined_error = float(np.hypot(formal_error, anchor_scatter / len(derived_estimates) ** 0.5)) + else: + combined_error = formal_error + + anchor_labels = [ + estimate['anchor_label'] for estimate in derived_estimates + if estimate.get('anchor_label') is not None + ] + derived_star = { + 'pos': selected_pos, + 'ra': selected_ra, + 'dec': selected_dec, + 'mag': combined_mag, + 'error': combined_error, + 'catalog_source': source_label, + 'is_aavso_vsp': False, + 'mag_band': mag_band or 'V', + 'observed_filter': observed_filter, + 'derived_catalog_reference': True, + 'derived_reference_anchor_count': len(derived_estimates), + 'derived_reference_anchor_labels': anchor_labels, + } + return f"{label_prefix}", derived_star + + +def preferred_catalog_magnitude_band_for_filter(observed_filter): + candidates = nextastro_photometry_band_candidates(observed_filter) + if not candidates: + return None + return candidates[0][2] + + +def catalog_band_priority(mag_band, observed_filter): + if observed_filter is None or str(observed_filter).strip() == '': + return 0 + preferred_band = preferred_catalog_magnitude_band_for_filter(observed_filter) + if preferred_band is None: + return 1 + return 0 if str(mag_band or '').strip().lower() == str(preferred_band).strip().lower() else 1 + + +def derived_catalog_reference_for_selected_comp(fit_lc_refs, comp_stars, vsp_comp_stars, vsp_ind, + best_comp, observed_filter=None, comp_ra_dec=None): + if best_comp is None or best_comp not in fit_lc_refs: + return None, None + + selected_ref = fit_lc_refs.get(best_comp) + if not isinstance(selected_ref, dict): + return None, None + selected_fit = selected_ref.get('myfit') + if selected_fit is None: + return None, None + selected_pos = comp_stars[best_comp] + selected_ra, selected_dec = None, None + if comp_ra_dec is not None and best_comp < len(comp_ra_dec): + selected_ra = _finite_float(comp_ra_dec[best_comp][0]) + selected_dec = _finite_float(comp_ra_dec[best_comp][1]) + derived_estimates = [] + + for anchor_index in vsp_ind: + if anchor_index == best_comp or anchor_index not in fit_lc_refs: + continue + if anchor_index < 0 or anchor_index >= len(comp_stars): + continue + + anchor_pos = comp_stars[anchor_index] + anchor_label = None + anchor_star = None + for label, star in vsp_comp_stars.items(): + if star.get('pos') == anchor_pos: + anchor_label = label + anchor_star = star + break + if anchor_star is None: + continue + + anchor_mag = _finite_float(anchor_star.get('mag')) + anchor_mag_error = normalized_magnitude_error(anchor_star.get('error')) + if catalog_band_priority(anchor_star.get('mag_band'), observed_filter) != 0: + continue + if ( + anchor_mag is None + or anchor_mag_error is None + or anchor_mag_error > CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX + or not is_usable_apparent_magnitude(anchor_mag) + ): + continue + + anchor_ref = fit_lc_refs.get(anchor_index) + if not isinstance(anchor_ref, dict): + continue + anchor_fit = anchor_ref.get('myfit') + if anchor_fit is None: + continue + + ratio = aligned_reference_curve_ratio(selected_fit, anchor_fit) + if ratio.size == 0: + continue + + with np.errstate(divide='ignore', invalid='ignore'): + selected_mag_points = anchor_mag - (2.5 * np.log10(ratio)) + selected_mag_points = selected_mag_points[ + np.isfinite(selected_mag_points) + & (selected_mag_points <= MAX_APPARENT_MAGNITUDE) + ] + if selected_mag_points.size == 0: + continue + + median_mag = float(np.nanmedian(selected_mag_points)) + mad = float(np.nanmedian(np.abs(selected_mag_points - median_mag))) + robust_scatter = 1.4826 * mad if np.isfinite(mad) else np.nan + if not np.isfinite(robust_scatter): + robust_scatter = float(np.nanstd(selected_mag_points)) + scatter_error = robust_scatter / max(selected_mag_points.size, 1) ** 0.5 + total_error = float(np.hypot(anchor_mag_error, scatter_error)) + if not np.isfinite(total_error) or total_error <= 0: + total_error = anchor_mag_error + + derived_estimates.append({ + 'mag': median_mag, + 'error': total_error, + 'mag_band': anchor_star.get('mag_band'), + 'anchor_label': anchor_label, + 'anchor_index': anchor_index, + 'points': int(selected_mag_points.size), + }) + + return combine_catalog_reference_estimates( + derived_estimates, + selected_pos, + observed_filter, + 'Derived from catalog-calibrated comparison stars', + mag_band=( + next((estimate.get('mag_band') for estimate in derived_estimates if estimate.get('mag_band')), None) + or 'V' + ), + label_prefix=f"Derived Comp {best_comp + 1}", + selected_ra=selected_ra, + selected_dec=selected_dec, + ) + + +def derive_selected_comp_catalog_reference_from_field(reference_image, wcs_file, field_catalog, comp_stars, + best_comp, observed_filter=None, + aperture_radius=REFERENCE_FALLBACK_DETECTION_APERTURE_RADIUS_PIXELS, + min_separation_pixels=REFERENCE_FALLBACK_DETECTION_MIN_SEP_PIXELS): + if ( + reference_image is None + or not wcs_file + or field_catalog is None + or best_comp is None + or best_comp < 0 + or best_comp >= len(comp_stars) + ): + return None, None + + selected_pos = comp_stars[best_comp] + selected_x = _finite_float(selected_pos[0]) + selected_y = _finite_float(selected_pos[1]) + if selected_x is None or selected_y is None: + return None, None + + selected_flux = image_aperture_signal_flux( + reference_image, + selected_x, + selected_y, + aperture_radius=aperture_radius, + ) + if not np.isfinite(selected_flux) or selected_flux <= 0: + return None, None + + try: + wcs_hdr = search_wcs(wcs_file) + except Exception: + return None, None + try: + selected_ra, selected_dec = wcs_hdr.pixel_to_world_values(selected_x, selected_y) + selected_ra = float(np.asarray(selected_ra).reshape(-1)[0]) + selected_dec = float(np.asarray(selected_dec).reshape(-1)[0]) + except Exception: + selected_ra, selected_dec = None, None + + image_shape = np.asarray(reference_image).shape + band_candidates = nextastro_photometry_band_candidates(observed_filter) + derived_estimates = [] + + for row in nextastro_catalog_rows(field_catalog): + row_ra = _finite_float(row.get('ra')) + row_dec = _finite_float(row.get('dec')) + if row_ra is None or row_dec is None: + continue + magnitude = row_nextastro_magnitude(row, band_candidates, max_error=MAX_APPARENT_MAGNITUDE) + if magnitude is None: + continue + if catalog_band_priority(magnitude.get('mag_band'), observed_filter) != 0: + continue + try: + anchor_x, anchor_y = wcs_hdr.world_to_pixel_values(row_ra, row_dec) + anchor_x = float(np.asarray(anchor_x).reshape(-1)[0]) + anchor_y = float(np.asarray(anchor_y).reshape(-1)[0]) + except Exception: + continue + if not pixel_within_image(anchor_x, anchor_y, image_shape, margin=float(aperture_radius) + 2.0): + continue + if (anchor_x - selected_x) ** 2 + (anchor_y - selected_y) ** 2 < float(min_separation_pixels) ** 2: + continue + + anchor_flux = image_aperture_signal_flux( + reference_image, + anchor_x, + anchor_y, + aperture_radius=aperture_radius, + ) + if not np.isfinite(anchor_flux) or anchor_flux <= 0: + continue + + with np.errstate(divide='ignore', invalid='ignore'): + selected_mag = magnitude['mag'] - (2.5 * np.log10(selected_flux / anchor_flux)) + if not is_usable_apparent_magnitude(selected_mag): + continue + + derived_estimates.append({ + 'mag': float(selected_mag), + 'error': float(magnitude['error']), + 'mag_band': magnitude.get('mag_band'), + 'anchor_label': ( + f"NextAstro-{row.get('source_id') or row.get('id')}" + if (row.get('source_id') or row.get('id')) not in (None, '') + else f"RA={row_ra:.6f} Dec={row_dec:.6f}" + ), + 'anchor_x': anchor_x, + 'anchor_y': anchor_y, + 'anchor_flux': float(anchor_flux), + }) + + label, star = combine_catalog_reference_estimates( + derived_estimates, + selected_pos, + observed_filter, + 'Derived from full-field catalog-calibrated stars', + mag_band=( + next((estimate.get('mag_band') for estimate in derived_estimates if estimate.get('mag_band')), None) + or 'V' + ), + label_prefix=f"Derived Field Comp {best_comp + 1}", + selected_ra=selected_ra, + selected_dec=selected_dec, + ) + return label, star + + +def catalog_reference_candidate_error(candidate): + if not isinstance(candidate, dict): + return np.inf + star = candidate.get('star') + if not isinstance(star, dict): + return np.inf + error = normalized_magnitude_error(star.get('error')) + return float(error) if error is not None and np.isfinite(error) else np.inf + + +def choose_selected_comp_catalog_reference_candidate(candidates, observed_filter=None): + usable = [ + candidate for candidate in candidates + if np.isfinite(catalog_reference_candidate_error(candidate)) + ] + if not usable: + return None + source_priority = { + 'direct_catalog': 0, + 'provided_comp_derived': 1, + 'field_derived': 2, + } + usable.sort(key=lambda candidate: ( + catalog_band_priority((candidate.get('star') or {}).get('mag_band'), observed_filter), + source_priority.get(candidate.get('source'), 3), + catalog_reference_candidate_error(candidate), + )) + return usable[0] + + +def stellar_variability(fit_lc_refs, fit_lc_best, comp_stars, vsp_comp_stars, vsp_ind, best_comp, save, s_name, + observed_filter=None, comp_ra_dec=None, field_catalog=None, reference_image=None, + wcs_file=None, observation_date=None, + catalog_match_radius_arcsec=NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC): + try: + if best_comp is None: + log_info( + "Skipping AID magnitude output because no transit-fit comparison star was selected.", + warn=True, + ) + return [] + comp_pos = comp_stars[best_comp] + selected_comp_fit = fit_lc_refs[best_comp]['myfit'] + except Exception as e: + log_info(f"Error selecting or calculating variability for comparison star: {e}", warn=True) + return [] + + candidates = [] + for key, value in vsp_comp_stars.items(): + if value.get('pos') == comp_pos: + candidates.append({ + 'label': key, + 'star': value, + 'source': 'direct_catalog', + 'error': value.get('error'), + }) + break + + direct_relaxed = build_direct_selected_catalog_candidate( + comp_stars, + comp_ra_dec, + field_catalog, + best_comp, + observed_filter=observed_filter, + match_radius_arcsec=catalog_match_radius_arcsec, + ) + if direct_relaxed is not None: + candidates.append(direct_relaxed) + + derived_label, derived_star = derived_catalog_reference_for_selected_comp( + fit_lc_refs, + comp_stars, + vsp_comp_stars, + vsp_ind, + best_comp, + observed_filter=observed_filter, + comp_ra_dec=comp_ra_dec, + ) + if derived_star is not None: + candidates.append({ + 'label': derived_label, + 'star': derived_star, + 'source': 'provided_comp_derived', + 'error': derived_star.get('error'), + }) + + has_preferred_band_candidate = any( + catalog_band_priority((candidate.get('star') or {}).get('mag_band'), observed_filter) == 0 + for candidate in candidates + ) + if not has_preferred_band_candidate: + field_label, field_star = derive_selected_comp_catalog_reference_from_field( + reference_image, + wcs_file, + field_catalog, + comp_stars, + best_comp, + observed_filter=observed_filter, + ) + if field_star is not None: + candidates.append({ + 'label': field_label, + 'star': field_star, + 'source': 'field_derived', + 'error': field_star.get('error'), + }) + + selected_candidate = choose_selected_comp_catalog_reference_candidate( + candidates, + observed_filter=observed_filter, + ) + if selected_candidate is None: + log_info( + "Skipping AID magnitude output because the transit-fit comparison star has no catalog " + f"magnitude and no derived magnitude could be inferred from calibrated comparison stars " + "or full-field catalog stars.", + warn=True, + ) + return [] + + comp_star = selected_candidate['star'] + vsp_auid_comp = selected_candidate['label'] + direct_error = catalog_reference_candidate_error( + next((candidate for candidate in candidates if candidate.get('source') == 'direct_catalog'), None) + ) + chosen_error = catalog_reference_candidate_error(selected_candidate) + if comp_star.get('derived_catalog_reference'): + log_info( + "AID magnitude output will use a derived catalog magnitude for the transit-fit comparison star " + f"from {comp_star.get('derived_reference_anchor_count', 0)} calibrated star(s): " + f"{comp_star.get('mag', np.nan):.3f} +/- {comp_star.get('error', np.nan):.3f} mag." + ) + elif np.isfinite(direct_error) and direct_error > CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX: + log_info( + "AID magnitude output will use the direct selected-comparison catalog magnitude even though " + f"its uncertainty exceeds {CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX:.3f} mag because it is no worse " + f"than the derived alternatives: {comp_star.get('mag', np.nan):.3f} +/- " + f"{chosen_error:.3f} mag." + ) + + try: + return build_stellar_variability_params_from_fit( + selected_comp_fit, + comp_star, + comp_pos, + vsp_auid_comp, + save, + s_name, + observed_filter=observed_filter, + observation_date=observation_date, + ) + except KeyError as e: + log_info(f"Key error in processing stellar variability: {e}", warn=True) + return [] + except Exception as e: + log_info(f"Error in processing stellar variability: {e}", warn=True) + return [] + + +# Mid-Transit Time Prior Helper Functions +def numberOfTransitsAway(timeData, period, originalT): + return int((np.nanmin(timeData) - originalT) / period) + 1 + + +def nearestTransitTime(timeData, period, originalT): + nearT = ((numberOfTransitsAway(timeData, period, originalT) * period) + originalT) + return nearT + + +def save_comp_ra_dec(wcs_file, ra_file, dec_file, comp_coords): + comp_ra, comp_dec = None, None + + if wcs_file: + comp_ra = ra_file[int(comp_coords[1])][int(comp_coords[0])] + comp_dec = dec_file[int(comp_coords[1])][int(comp_coords[0])] + + comp_star = { + 'ra': str(comp_ra) if comp_ra else comp_ra, + 'dec': str(comp_dec) if comp_dec else comp_dec, + 'x': str(comp_coords[0]) if comp_coords[0] else comp_coords[0], + 'y': str(comp_coords[1]) if comp_coords[1] else comp_coords[1] + } + + return comp_star + + +def realTimeReduce(i, target_name, p_dict, info_dict, ax, use_nextastro_astrometry=False, multiprocess_transformations=None): + timeList, airMassList, exptimes, norm_flux = [], [], [], [] + ignore_header_wcs = should_ignore_header_wcs(info_dict.get('ignore_header_wcs')) + allow_pixel_alignment_fallback = should_allow_pixel_alignment_fallback( + info_dict.get('allow_pixel_alignment_fallback', True) + ) + pixel_alignment_enabled = bool(ignore_header_wcs or allow_pixel_alignment_fallback) + if ignore_header_wcs: + log_info("Pixel alignment enabled explicitly: header WCS will be ignored for manual alignment.") + elif allow_pixel_alignment_fallback: + log_info( + "WCS coverage-aware coordinate mode enabled: per-frame WCS is preferred when coverage is " + "consistent, with pixel alignment fallback available for incomplete-WCS datasets." + ) + else: + log_info( + "WCS-authoritative coordinate mode enabled: each frame's header WCS will supply star pixel " + "positions; Astroalign/pixel alignment fallback is disabled." + ) + bad_wcs_threshold_fraction = get_bad_wcs_threshold_fraction(info_dict.get('bad_wcs_threshold_percent')) + pointing_rejection_sigma = get_pointing_rejection_sigma(info_dict.get('pointing_rejection_sigma')) + detect_bad_pixels_before_photometry = should_detect_bad_pixels_before_photometry( + info_dict.get('detect_bad_pixels_before_photometry', 'n') + ) + multiprocess_bad_pixel_precheck = get_multiprocess_bad_pixel_precheck_processes( + info_dict.get('multiprocess_bad_pixel_precheck', 'n') + ) + + plateStatus.initializeFilenames(info_dict['images']) + inputfiles = corruption_check(info_dict['images']) + if not ignore_header_wcs: + maybe_reinterpret_decimal_ra_hours_from_wcs(inputfiles, p_dict) + # time sort images + times = [] + for ifile in inputfiles: + plateStatus.setCurrentFilename(ifile) + extension = 0 + header = fits.getheader(filename=ifile, ext=extension) + while header['NAXIS'] == 0: + extension += 1 + header = fits.getheader(filename=ifile, ext=extension) + obsTime = img_time_bjd_tdb(header, p_dict, info_dict) + times.append(obsTime) + plateStatus.setObsTime(obsTime) + + si = np.argsort(times) + times = np.array(times)[si] + inputfiles = np.array(inputfiles)[si] + inputfiles, wcs_keep_mask, dropped_wcs_files = filter_sparse_missing_wcs_frames( + inputfiles, + ignore_header_wcs=ignore_header_wcs, + max_missing_fraction=bad_wcs_threshold_fraction, + allow_pixel_alignment_fallback=allow_pixel_alignment_fallback, + ) + if dropped_wcs_files: + times = times[wcs_keep_mask] + plateStatus.initializeFilenames(list(inputfiles)) + if len(inputfiles) == 0: + log_info( + "Error: no input frame has celestial WCS and pixel alignment fallback is disabled.", + error=True, + ) + return + target_wcs_precheck_inputfiles = np.array(inputfiles, copy=True) + target_wcs_reference_file = inputfiles[0] if len(inputfiles) else None + inputfiles, target_wcs_keep_mask, dropped_target_wcs_files = filter_wcs_target_out_of_frame_frames( + inputfiles, + p_dict, + obs_times=times, + ignore_header_wcs=ignore_header_wcs, + ) + if dropped_target_wcs_files: + target_reference_fallback = reference_frame_rejection_fallback_info( + target_wcs_reference_file, + dropped_target_wcs_files, + ordered_inputfiles=target_wcs_precheck_inputfiles, + rejection_label="Target WCS precheck", + ) + times = times[target_wcs_keep_mask] + plateStatus.initializeFilenames(list(inputfiles)) + else: + target_reference_fallback = None + if len(inputfiles) == 0: + log_info( + "Error: target WCS precheck removed every frame because the target RA/Dec projects outside " + "each image.", + error=True, + ) + return + pointing_precheck_inputfiles = np.array(inputfiles, copy=True) + pointing_reference_file = inputfiles[0] if len(inputfiles) else None + inputfiles, _, dropped_pointing_files, pointing_alignment_transforms = filter_pointing_outlier_frames( + inputfiles, + pointing_rejection_sigma=pointing_rejection_sigma, + ignore_header_wcs=ignore_header_wcs, + allow_pixel_alignment_fallback=allow_pixel_alignment_fallback, + return_alignment_transforms=True, + multiprocess_transformations=multiprocess_transformations, + ) + if dropped_pointing_files: + pointing_reference_fallback = reference_frame_rejection_fallback_info( + pointing_reference_file, + dropped_pointing_files, + ordered_inputfiles=pointing_precheck_inputfiles, + ) + reference_fallback = pointing_reference_fallback or target_reference_fallback + plateStatus.initializeFilenames(list(inputfiles)) + else: + reference_fallback = target_reference_fallback + if reference_fallback is not None and reference_fallback.get('next_reference_candidate') is None: + log_info( + "Error: all leading reference candidates were rejected by the pointing precheck; no usable " + "realtime reference image remains.", + error=True, + ) + return + if reference_fallback is not None: + pointing_alignment_transforms = {} + + bad_pixel_reference = None + if detect_bad_pixels_before_photometry: + log_info( + "Bad-pixel precheck enabled: scanning frames for persistent isolated high-count outliers before photometry." + ) + bad_pixel_reference = build_persistent_bad_pixel_map( + inputfiles, + load_image_data, + save_directory=info_dict['save'], + max_processes=multiprocess_bad_pixel_precheck, + ) + else: + log_info("Bad-pixel precheck disabled per optional_info setting.") + + exotic_UIprevTPX = info_dict['tar_coords'][0] + exotic_UIprevTPY = info_dict['tar_coords'][1] + + plateStatus.setCurrentFilename(inputfiles[0]) + header = get_first_image_header(inputfiles[0]) + wcs_file = check_wcs(inputfiles[0], info_dict['save'], info_dict['plate_opt'], rt=True, + use_nextastro_astrometry=use_nextastro_astrometry, + ra=p_dict.get('ra'), dec=p_dict.get('dec'), pixel_scale=info_dict.get('pixel_scale'), + ignore_header_wcs=ignore_header_wcs) + comp_star = info_dict['comp_stars'] + tar_radec, comp_radec = None, [] + first_image = fits.getdata(inputfiles[0]) + + if wcs_file: + wcs_header = get_first_image_header(wcs_file) + + ra_file, dec_file = get_ra_dec(wcs_header, image_shape=first_image.shape) + if reference_fallback is not None: + target_projection = estimate_target_pixel_from_ra_dec( + p_dict, + wcs_header, + first_image, + timeList[0] if timeList else img_time_bjd_tdb(header, p_dict, info_dict), + ) + if target_projection is None: + log_info( + "Error: could not estimate target coordinates from RA/Dec after removing the rejected " + "reference image.", + error=True, + ) + return + exotic_UIprevTPX, exotic_UIprevTPY, target_ra, target_dec = target_projection + info_dict['tar_coords'] = [exotic_UIprevTPX, exotic_UIprevTPY] + tar_radec = (target_ra, target_dec) + fallback_comp_stars, fallback_candidates = select_reference_fallback_comparison_stars( + first_image, + first_image.shape, + [exotic_UIprevTPX, exotic_UIprevTPY], + comp_count=1, + ) + log_reference_fallback_comparison_candidates( + fallback_comp_stars, + fallback_candidates, + ) + if fallback_comp_stars: + comp_star = fallback_comp_stars[0] + info_dict['comp_stars'] = comp_star + else: + log_info( + "Error: no replacement image-detected comparison star was available after removing " + "the rejected realtime reference image.", + error=True, + ) + return + else: + tar_radec = (ra_file[int(exotic_UIprevTPY)][int(exotic_UIprevTPX)], + dec_file[int(exotic_UIprevTPY)][int(exotic_UIprevTPX)]) + + ra = ra_file[int(comp_star[1])][int(comp_star[0])] + dec = dec_file[int(comp_star[1])][int(comp_star[0])] + + comp_radec.append((ra, dec)) + elif reference_fallback is not None: + log_info( + "Error: the original realtime reference image was removed, but the new reference image does not " + "have a usable WCS. EXOTIC cannot estimate target coordinates from RA/Dec or choose a replacement " + "image-detected comparison star without a new reference WCS.", + error=True, + ) + return + + target_and_comp_radec = None + if tar_radec is not None and comp_radec: + target_and_comp_radec = np.array([tar_radec, comp_radec[0]], dtype=float) + target_and_comp_pixels = np.array( + [[exotic_UIprevTPX, exotic_UIprevTPY], comp_star], + dtype=float, + ) + + centroid_reference_image = load_image_data(inputfiles[0]) + centroid_reference_image = repair_bad_pixels_in_frame(centroid_reference_image, bad_pixel_reference) + targ_sig_xy = fit_centroid(centroid_reference_image, [exotic_UIprevTPX, exotic_UIprevTPY], 0)[3:5] + del centroid_reference_image + + # aperture and annulus scale factors in PSF sigma units + aper_sigma = finite_positive_or_nan(3 * max(targ_sig_xy)) + if not np.isfinite(aper_sigma): + aper_sigma = 3.0 + aper_sigma = float(np.clip(aper_sigma, APERTURE_SIGMA_MIN, APERTURE_SIGMA_MAX)) + annulus_sigma = 10 + fast_aperture_mask = is_fast_aperture_mask_enabled(info_dict.get('fast_aperture_mask')) + use_adaptive_apertures = is_adaptive_aperture_mode_enabled(info_dict.get('use_adaptive_apertures')) + use_aperture_corrections_and_full_image_fwhm = should_use_aperture_corrections_and_full_image_fwhm( + info_dict.get('use_aperture_corrections_and_full_image_fwhm', False) + ) + aper = np.nan + annulus = np.nan + sigma = np.nan + if use_adaptive_apertures: + log_info( + "Adaptive aperture scaling enabled for realtime photometry: " + f"aperture scales are in PSF sigma units (1 image FWHM = {GAUSSIAN_SIGMA_TO_FWHM:.3f} sigma)." + ) + log_info( + "Realtime aperture candidates are limited to " + f"{APERTURE_MIN_FWHM_MULTIPLIER:.1f}-{APERTURE_MAX_FWHM_MULTIPLIER:.1f} image FWHM " + f"({APERTURE_SIGMA_MIN:.2f}-{APERTURE_SIGMA_MAX:.2f} sigma)." + ) + if use_aperture_corrections_and_full_image_fwhm: + log_info("Aperture corrections and full-image FWHM estimation enabled for realtime photometry.") + + # alloc psf fitting param + psf_data = { + # x-cent, y-cent, amplitude, sigma-x, sigma-y, rotation, offset + 'target': np.zeros((len(inputfiles), 7)), # PSF fit + 'comp': np.zeros((len(inputfiles), 7)) + } + tar_comp_dist = { + 'comp': np.zeros(2, dtype=int) + } + + # open files, calibrate, align, photometry + reset_transform_timing_stats() + reset_photometry_timing_stats() + multiprocess_alignment_results = None + use_multiprocess_alignment = multiprocess_transformations is not None and multiprocess_transformations > 0 + if use_multiprocess_alignment: + multiprocess_alignment_results = build_multiprocess_alignment_results( + inputfiles, + multiprocess_transformations, + target_and_comp_pixels, + target_and_comp_radec=target_and_comp_radec, + ignore_header_wcs=ignore_header_wcs, + bad_pixel_reference=bad_pixel_reference, + use_fast_centroid_cadence=True, + use_adaptive_apertures=use_adaptive_apertures, + compute_fallback_transform=pixel_alignment_enabled, + first_frame_uses_input_comp_pixels=True, + precomputed_fallback_transforms=pointing_alignment_transforms, + ) + use_multiprocess_transform_precompute = False + fallback_transforms = pointing_alignment_transforms + for i, fileName in enumerate(inputfiles): + plateStatus.setCurrentFilename(fileName) + hdul = fits.open(name=fileName, memmap=False, cache=False, lazy_load_hdus=False, + ignore_missing_end=True) + frame_fast_centroid = should_use_fast_centroid(i) + target_fast_centroid = should_use_fast_target_centroid(i, adaptive_apertures=use_adaptive_apertures) + + extension = 0 + image_header = hdul[extension].header + while image_header["NAXIS"] == 0: + extension += 1 + image_header = hdul[extension].header + + # TIME + timeVal = img_time_bjd_tdb(image_header, p_dict, info_dict) + timeList.append(timeVal) + + # IMAGES + imageData = hdul[extension].data + imageData = repair_bad_pixels_in_frame(imageData, bad_pixel_reference) + + if i == 0: + firstImage = np.copy(imageData) + + if multiprocess_alignment_results is not None: + apply_parallel_alignment_result( + multiprocess_alignment_results[i], + i, + psf_data, + tar_comp_dist, + ['comp'], + ) + else: + alignment_result = { + 'index': i, + 'file_name': fileName, + 'wcs': None, + 'fallback': None, + } + previous_psf_rows = {} + if i != 0: + previous_psf_rows = { + 'target': psf_data['target'][i - 1], + 'comp1': psf_data['comp'][i - 1], + } + + has_wcs_alignment = False + if not ignore_header_wcs: + try: + wcs_hdr = search_wcs_from_header(image_header) + has_wcs_alignment = wcs_hdr.is_celestial + except Exception: + has_wcs_alignment = False + + if has_wcs_alignment and target_and_comp_radec is not None: + try: + pix_x, pix_y = wcs_hdr.world_to_pixel_values( + target_and_comp_radec[:, 0], + target_and_comp_radec[:, 1], + ) + pix_x = np.asarray(pix_x, dtype=float).reshape(-1) + pix_y = np.asarray(pix_y, dtype=float).reshape(-1) + projected_coords = np.column_stack((pix_x, pix_y)) + + wcs_candidate = _fit_alignment_candidate_psfs( + imageData, + projected_coords, + target_fast_centroid, + frame_fast_centroid, + ) + wcs_candidate['projected_off_frame'] = any_projected_coord_out_of_frame( + projected_coords, + imageData.shape, + ) + alignment_result['wcs'] = wcs_candidate + except Exception: + alignment_result['wcs'] = None + + log_alignment_progress( + i, + len(inputfiles), + fileName, + use_multiprocess_transform_precompute, + pixel_alignment_enabled=pixel_alignment_enabled, + ) + + wcs_candidate_acceptable = wcs_alignment_candidate_is_acceptable( + alignment_result, + i, + psf_data, + tar_comp_dist, + ['comp'], + ) + if not pixel_alignment_enabled and not wcs_candidate_acceptable: + log_wcs_authoritative_candidate_diagnostics( + alignment_result, + i, + psf_data, + tar_comp_dist, + ['comp'], + ) + if pixel_alignment_enabled and not wcs_candidate_acceptable: + cached_tform = fallback_transforms.get(str(fileName)) if fallback_transforms else None + if cached_tform is not None: + tform = cached_tform + elif i == 0: + tform = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) else: - userpdict[key] = user_input(f"Enter the {planet_params[i]}: ", type_=type(userpdict[key])) - # Did not use initialization file or null + tform = downsampled_fallback_transformation( + imageData, + fileName, + reference_image=firstImage, + ) + + transformed_coords = np.asarray(tform(target_and_comp_pixels), dtype=float) + alignment_result['fallback'] = _fit_alignment_candidate_psfs( + imageData, + transformed_coords, + target_fast_centroid, + frame_fast_centroid, + previous_psf_rows=previous_psf_rows, + ) + apply_parallel_alignment_result( + alignment_result, + i, + psf_data, + tar_comp_dist, + ['comp'], + ) + + # aperture photometry + target_sigma = psf_sigma_from_fit(psf_data['target'][i], fallback_sigma=sigma) + target_fwhm = psf_fwhm_from_sigma(target_sigma) + field_star_psfs = np.empty((0, 7), dtype=float) + image_fwhm = target_fwhm + if use_aperture_corrections_and_full_image_fwhm: + field_star_psfs = estimate_isolated_field_star_psfs(imageData, fwhm_hint=target_fwhm) + image_fwhm = image_fwhm_from_field_star_psfs(field_star_psfs, fallback_fwhm=target_fwhm) + frame_sigma = ( + image_fwhm / GAUSSIAN_SIGMA_TO_FWHM + if np.isfinite(image_fwhm) and image_fwhm > 0 + else target_sigma + ) + if i == 0: + sigma = frame_sigma + if not np.isfinite(sigma) or sigma <= 0: + log_info("Warning: Initial PSF sigma is invalid; using sigma=1.0 for aperture photometry.", warn=True) + sigma = 1.0 + + if use_adaptive_apertures: + frame_aper, frame_annulus = resolve_frame_aperture_radii( + [aper_sigma], + [annulus_sigma], + adaptive_apertures=True, + frame_sigma=frame_sigma, + fallback_sigma=sigma, + ) + aper = float(frame_aper[0]) + annulus = float(frame_annulus[0]) + elif i == 0: + aper, annulus = resolve_frame_aperture_radii( + [aper_sigma], + [annulus_sigma], + adaptive_apertures=True, + frame_sigma=sigma, + fallback_sigma=sigma, + ) + aper = float(aper[0]) + annulus = float(annulus[0]) + + aperture_correction_factor = 1.0 + if use_aperture_corrections_and_full_image_fwhm: + aperture_correction = build_aperture_correction_profile( + imageData, + [aper], + fwhm_hint=image_fwhm, + fast_mode=fast_aperture_mask, + field_star_psfs=field_star_psfs, + ) + correction_factors = np.asarray(aperture_correction.get('correction_factors', [1.0]), dtype=float).reshape(-1) + if correction_factors.size and np.isfinite(correction_factors[0]) and correction_factors[0] > 0: + aperture_correction_factor = float(correction_factors[0]) + + comp_frame_sigma = psf_sigma_from_fit(psf_data['comp'][i], fallback_sigma=frame_sigma) + tFlux = aperPhot( + imageData, + 0, + psf_data['target'][i, 0], + psf_data['target'][i, 1], + aper, + annulus, + fast_mode=fast_aperture_mask, + sigma_hint=frame_sigma, + )[0] * aperture_correction_factor + cFlux = aperPhot( + imageData, + 1, + psf_data['comp'][i, 0], + psf_data['comp'][i, 1], + aper, + annulus, + fast_mode=fast_aperture_mask, + sigma_hint=comp_frame_sigma, + )[0] * aperture_correction_factor + norm_flux.append(tFlux / cFlux) + + # close file + delete from memory + hdul.close() + del hdul + # Replaced each loop, so clean up + del imageData + + log_transform_timing_stats('Transformation timing summary (real-time reduce)') + log_photometry_timing_stats('Photometry timing summary (real-time reduce)') + log_reduction_timing_overview('Reduction timing overview (real-time reduce)') + + ax.clear() + ax.set_title(target_name) + ax.set_ylabel('Normalized Flux') + ax.set_xlabel('Time (JD)') + ax.plot(timeList, norm_flux, 'bo') + + +def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None, + allow_mid_transit_range_warning=True, disable_vertical_flux_normalization=False, + final_fit_mode='lm', + use_impactparameter_rather_than_inclination_to_fit=True, + plot_time_range=None, + use_eebls_to_initialize_tmid_and_bounds=True, + compute_eebls_diagnostics=False, + target_flux_error=None, + comp_flux_error=None, + exposure_times_seconds=None, + gain_e_per_adu=None): + plot_time_range = np.asarray(times if plot_time_range is None else plot_time_range, dtype=float) + prepared = prepare_lightcurve_fit_input_series( + times, + tFlux, + cFlux, + airmass, + target_flux_error=target_flux_error, + comp_flux_error=comp_flux_error, + jd_times=jd_times, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=gain_e_per_adu, + expected_transit_depth=expected_transit_depth_from_planet_dict(pDict), + ) + if not prepared.get('applied'): + return None, None, None + + filter_diagnostics = prepared['filter_diagnostics'] + debug_times = prepared['debug_times'] + debug_target_flux = prepared['debug_target_flux'] + debug_comp_flux = prepared['debug_comp_flux'] + debug_raw_ratio = prepared['debug_raw_ratio'] + debug_target_flux_error = prepared.get('debug_target_flux_error') + debug_comp_flux_error = prepared.get('debug_comp_flux_error') + debug_relative_flux_error = prepared.get('debug_relative_flux_error') + debug_initial_sigma_keep_mask = prepared['initial_sigma_keep_mask'] + debug_prefit_raw_ratio_keep_mask = prepared['prefit_raw_ratio_keep_mask'] + arrayFinalFlux = prepared['flux'] + f1 = prepared['target_flux'] + f2 = prepared['comp_flux'] + f1_error = np.asarray(prepared.get('target_flux_error'), dtype=float) + f2_error = np.asarray(prepared.get('comp_flux_error'), dtype=float) + arrayNormUnc = prepared['unc'] + arrayTimes = prepared['time'] + arrayJDTimes = prepared['jd_time'] + arrayExposureTimes = prepared.get('exposure_time_seconds') + arrayExposureTimes = None if arrayExposureTimes is None else np.asarray(arrayExposureTimes, dtype=float) + arrayAirmass = prepared['airmass'] + skip_airmass_fit = prepared['skip_airmass_fit'] + + + # -----LM LIGHTCURVE FIT-------------------------------------- + prior = { + 'rprs': pDict['rprs'], # Rp/Rs + 'ars': pDict['aRs'], # a/Rs + 'per': pDict['pPer'], # Period [day] + 'inc': pDict['inc'], # Inclination [deg] + 'u0': ld[0], 'u1': ld[1], 'u2': ld[2], 'u3': ld[3], # limb darkening (nonlinear) + 'ecc': pDict['ecc'], # Eccentricity + 'omega': pDict['omega'], # Arg of periastron + 'tmid': pDict['midT'], # time of mid transit [day] + 'a2': 0, # Flux lower bound + } + + arrayPhases = (arrayTimes - pDict['midT']) / prior['per'] + expected_duration = estimate_transit_duration_from_prior_geometry(prior) + tmid_search_summary = estimate_ephemeris_tmid_and_bounds( + arrayTimes, + pDict['midT'], + prior['per'], + pDict['midTUnc'], + pDict['pPerUnc'], + expected_duration=expected_duration, + sigma_multiplier=25.0, + ) + prior['tmid'] = tmid_search_summary['tmid'] + lower, upper = tmid_search_summary['bounds'] + ephemeris_tmid_search_summary = tmid_search_summary + + if ( + allow_mid_transit_range_warning + and np.floor(arrayPhases).max() - np.floor(arrayPhases).min() == 0 + ): + log_mid_transit_range_warning_once(arrayTimes, prior['tmid']) + + if tmid_search_summary.get('duration_capped'): + log_info(tmid_search_summary['note']) + eebls_search_summary = None + if use_eebls_to_initialize_tmid_and_bounds or compute_eebls_diagnostics: + eebls_search_summary = estimate_tmid_and_bounds_with_eebls( + arrayTimes, + arrayFinalFlux, + arrayNormUnc, + prior, + [lower, upper], + ) + if use_eebls_to_initialize_tmid_and_bounds and eebls_search_summary.get('applied'): + tmid_search_summary = eebls_search_summary + prior['tmid'] = tmid_search_summary['tmid'] + lower, upper = tmid_search_summary['bounds'] + + search_restriction_prior = enrich_search_restriction_prior_with_rprs_data_uncertainty( + build_search_restriction_prior_from_planet_dict(pDict), + arrayTimes, + arrayFinalFlux, + arrayNormUnc, + prior, + context_label="initial light curve", + ) + mybounds = build_initial_transit_bounds( + prior, + [lower, upper], + ars_unc=pDict.get('aRsUnc'), + rprs_data_uncertainty=search_restriction_prior.get('rprs_data_uncertainty'), + search_restriction_prior=search_restriction_prior, + ) + apply_vertical_flux_normalization_bound( + prior, + mybounds, + arrayFinalFlux, + disable_vertical_flux_normalization, + ) + if not skip_airmass_fit: + mybounds['a2'] = [-1, 1] + + if arrayTimes.shape[0] < LIGHTCURVE_MIN_VALID_POINTS: + return None, None, None + + if np.isnan(arrayTimes).any() or np.isnan(arrayFinalFlux).any() or np.isnan(arrayNormUnc).any(): + log_info("\nWarning: NANs in time, flux or error", warn=True) + + fit_kwargs = { + 'jd_times': arrayJDTimes, + 'mode': 'lm', + 'use_impactparameter_rather_than_inclination_to_fit': + use_impactparameter_rather_than_inclination_to_fit, + } + add_exposure_times_to_lc_fitter_kwargs(fit_kwargs, arrayExposureTimes) + myfit = lc_fitter( + arrayTimes, + arrayFinalFlux, + arrayNormUnc, + arrayAirmass, + prior, + mybounds, + **fit_kwargs, + ) + myfit = apply_plot_time_range(myfit, plot_time_range) + annotate_airmass_fit(myfit, arrayAirmass, skip_airmass_fit) + annotate_lightcurve_filter_diagnostics(myfit, filter_diagnostics) + annotate_lightcurve_tmid_search(myfit, tmid_search_summary) + annotate_lightcurve_eebls_diagnostic(myfit, eebls_search_summary) + + if ( + myfit is not None + and hasattr(myfit, 'residuals') + and hasattr(myfit, 'phase') + and np.shape(myfit.residuals) == np.shape(arrayTimes) + and np.shape(myfit.phase) == np.shape(arrayTimes) + ): + phase_clip_mask = phase_bin_sigma_clip(myfit.residuals, myfit.phase, sigma=3, bins=10) + min_required_points = max(len(mybounds) + 1, 5) + if np.count_nonzero(~phase_clip_mask) >= min_required_points and np.any(phase_clip_mask): + filter_diagnostics.append(build_time_rejection_diagnostic( + "Phase-binned residual clip", + arrayTimes, + ~phase_clip_mask, + note="Dropped phase-binned residual outliers after the initial LM fit before refitting.", + )) + arrayFinalFlux = arrayFinalFlux[~phase_clip_mask] + arrayNormUnc = arrayNormUnc[~phase_clip_mask] + arrayTimes = arrayTimes[~phase_clip_mask] + arrayJDTimes = arrayJDTimes[~phase_clip_mask] + if arrayExposureTimes is not None: + arrayExposureTimes = arrayExposureTimes[~phase_clip_mask] + arrayAirmass = arrayAirmass[~phase_clip_mask] + f1 = f1[~phase_clip_mask] + f2 = f2[~phase_clip_mask] + f1_error = f1_error[~phase_clip_mask] + f2_error = f2_error[~phase_clip_mask] + + fit_kwargs = { + 'jd_times': arrayJDTimes, + 'mode': 'lm', + 'use_impactparameter_rather_than_inclination_to_fit': + use_impactparameter_rather_than_inclination_to_fit, + } + add_exposure_times_to_lc_fitter_kwargs(fit_kwargs, arrayExposureTimes) + myfit = lc_fitter( + arrayTimes, + arrayFinalFlux, + arrayNormUnc, + arrayAirmass, + prior, + mybounds, + **fit_kwargs, + ) + myfit = apply_plot_time_range(myfit, plot_time_range) + annotate_airmass_fit(myfit, arrayAirmass, skip_airmass_fit) + annotate_lightcurve_filter_diagnostics(myfit, filter_diagnostics) + annotate_lightcurve_tmid_search(myfit, tmid_search_summary) + annotate_lightcurve_eebls_diagnostic(myfit, eebls_search_summary) + + debug_prefit_keep_mask = debug_initial_sigma_keep_mask & debug_prefit_raw_ratio_keep_mask + debug_phase_clip_keep_mask = np.ones(np.count_nonzero(debug_prefit_keep_mask), dtype=bool) + if final_fit_mode == 'ns' and myfit is not None: + duration_prior = build_single_transit_duration_prior(pDict) + pre_ultranest_coverage_assessment = build_expected_transit_coverage_assessment( + arrayTimes, + prior, + flux_values=arrayFinalFlux, + flux_errors=arrayNormUnc, + tmid_search_summary=ephemeris_tmid_search_summary, + duration_prior=duration_prior, + ) + log_expected_transit_coverage_assessment(pre_ultranest_coverage_assessment) + nested_refinement = build_nested_tmid_refinement_from_initial_fit( + arrayTimes, + arrayFinalFlux, + arrayNormUnc, + prior, + mybounds, + myfit, + ) + myfit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + arrayTimes, + arrayFinalFlux, + arrayNormUnc, + arrayAirmass, + nested_refinement['prior'], + nested_refinement['bounds'], + jd_times=arrayJDTimes, + exposure_times_seconds=arrayExposureTimes, + use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + duration_prior=duration_prior, + search_restriction_prior=search_restriction_prior, + ) + annotate_pre_ultranest_transit_coverage(myfit, pre_ultranest_coverage_assessment) + myfit = apply_plot_time_range(myfit, plot_time_range) + annotate_airmass_fit(myfit, arrayAirmass, skip_airmass_fit) + annotate_lightcurve_filter_diagnostics(myfit, filter_diagnostics) + if myfit is not None: + annotate_lightcurve_tmid_search(myfit, tmid_search_summary) + annotate_lightcurve_eebls_diagnostic(myfit, eebls_search_summary) + annotate_nested_tmid_refinement( + myfit, + nested_refinement.get('applied', False), + note=nested_refinement.get('note'), + original_tmid_bounds=nested_refinement.get('original_tmid_bounds'), + refined_tmid_bounds=nested_refinement.get('refined_tmid_bounds'), + ) + + if myfit is not None: + if 'phase_clip_mask' in locals(): + debug_phase_clip_keep_mask = np.asarray(~phase_clip_mask, dtype=bool).copy() + annotate_selected_photometry_debug( + myfit, + debug_times, + debug_target_flux, + debug_comp_flux, + debug_raw_ratio, + debug_initial_sigma_keep_mask, + target_flux_error=debug_target_flux_error, + comp_flux_error=debug_comp_flux_error, + relative_flux_error=debug_relative_flux_error, + prefit_raw_ratio_keep_mask=debug_prefit_raw_ratio_keep_mask, + phase_clip_keep_mask_on_sigma_filtered=debug_phase_clip_keep_mask, + ) + annotate_transit_qc_expected_values(myfit, pDict) + annotate_transit_detection_qc(myfit) + annotate_stellar_variability_raw_photometry( + myfit, + f1, + f2, + target_flux_error=f1_error, + comp_flux_error=f2_error, + ) + + return myfit, f1, f2 + + +def diagnose_lightcurve_fit_inputs( + times, + tflux, + cflux, + airmass, + target_flux_error=None, + comp_flux_error=None, + enforce_relative_flux_max=True, + expected_transit_depth=None, +): + times = np.asarray(times, dtype=float) + tflux = np.asarray(tflux, dtype=float) + cflux = np.asarray(cflux, dtype=float) + airmass = np.asarray(airmass, dtype=float) + target_flux_error = valid_flux_error_array(target_flux_error, tflux.shape) + comp_flux_error = valid_flux_error_array(comp_flux_error, cflux.shape) + + diagnostics = { + 'input_point_count': int(times.shape[0]), + 'has_reference_flux': False, + 'relative_flux_point_count': 0, + 'sigma_clip_point_count': 0, + 'prefit_raw_ratio_clip_point_count': 0, + 'usable_point_count': 0, + 'failed_stage': None, + 'failure_reason': None, + } + + if diagnostics['input_point_count'] <= 1: + diagnostics.update({ + 'relative_flux_point_count': diagnostics['input_point_count'], + 'failed_stage': 'coverage', + 'failure_reason': ( + f"only {diagnostics['input_point_count']} frame(s) remained after masking invalid " + "comparison flux; need at least 2 to fit." + ), + }) + return diagnostics + + si = np.argsort(times) + times_sorted = times[si] + tflux_sorted = tflux[si] + cflux_sorted = cflux[si] + target_flux_error_sorted = None if target_flux_error is None else target_flux_error[si] + comp_flux_error_sorted = None if comp_flux_error is None else comp_flux_error[si] + with np.errstate(divide='ignore', invalid='ignore'): + flux_ratio_sorted = np.divide(tflux_sorted, cflux_sorted) + + has_reference_flux = not np.allclose(cflux_sorted, 1.0) + diagnostics['has_reference_flux'] = bool(has_reference_flux) + diagnostics['relative_flux_point_count'] = int(times_sorted.shape[0]) + + if has_reference_flux: + finite_ratio_mask = np.isfinite(flux_ratio_sorted) + nonpositive_ratio_mask = finite_ratio_mask & np.less_equal(flux_ratio_sorted, 0) + finite_ratio_values = flux_ratio_sorted[finite_ratio_mask] + ratio_range_text = "finite ratio range=n/a" + if finite_ratio_values.size: + ratio_range_text = ( + f"finite ratio range={np.nanmin(finite_ratio_values):.4f} to " + f"{np.nanmax(finite_ratio_values):.4f}" + ) + nonfinite_ratio_count = int(np.count_nonzero(~finite_ratio_mask)) + nonpositive_ratio_count = int(np.count_nonzero(nonpositive_ratio_mask)) + rejected_ratio_count = nonfinite_ratio_count + nonpositive_ratio_count + relative_flux_mask = valid_flux_ratio_mask(flux_ratio_sorted) + rejection_detail = ( + f"(non-finite={nonfinite_ratio_count}, non-positive={nonpositive_ratio_count}, " + f"{ratio_range_text})." + ) + rejection_context = "invalid target/reference ratio screening" + diagnostics['relative_flux_point_count'] = int(np.count_nonzero(relative_flux_mask)) + times_sorted = times_sorted[relative_flux_mask] + tflux_sorted = tflux_sorted[relative_flux_mask] + cflux_sorted = cflux_sorted[relative_flux_mask] + if target_flux_error_sorted is not None: + target_flux_error_sorted = target_flux_error_sorted[relative_flux_mask] + if comp_flux_error_sorted is not None: + comp_flux_error_sorted = comp_flux_error_sorted[relative_flux_mask] + flux_ratio_sorted = flux_ratio_sorted[relative_flux_mask] + airmass_sorted = airmass[si][relative_flux_mask] + if diagnostics['relative_flux_point_count'] <= 1: + diagnostics.update({ + 'failed_stage': 'relative_flux_filter', + 'failure_reason': ( + "relative-flux filtering left " + f"{diagnostics['relative_flux_point_count']} usable point(s); " + f"rejected {rejected_ratio_count}/{diagnostics['input_point_count']} frame(s) " + f"during {rejection_context} {rejection_detail}" + ), + }) + return diagnostics + else: + airmass_sorted = airmass[si] + + dt = np.mean(np.diff(times_sorted)) + if np.isfinite(dt) and dt > 0: + ndt = int(25. / 24. / 60. / dt) * 2 + 1 + else: + ndt = 5 + if ndt > len(times_sorted): + ndt = int(len(times_sorted) / 4) * 2 + 1 + filtered_data = sigma_clip(flux_ratio_sorted, sigma=3, dt=max(5, ndt), times=times_sorted) + valid_mask = ~filtered_data + diagnostics['sigma_clip_point_count'] = int(np.count_nonzero(valid_mask)) + if diagnostics['sigma_clip_point_count'] <= 1: + diagnostics.update({ + 'failed_stage': 'sigma_clip', + 'failure_reason': ( + "sigma clipping left " + f"{diagnostics['sigma_clip_point_count']} usable point(s); not enough data remained " + "for a lightcurve fit." + ), + }) + return diagnostics + + sigma_kept_indices = np.flatnonzero(valid_mask) + if sigma_kept_indices.size: + raw_ratio_outlier_mask = prefit_raw_ratio_outlier_mask( + flux_ratio_sorted[sigma_kept_indices], + times=times_sorted[sigma_kept_indices], + expected_transit_depth=expected_transit_depth, + use_global=has_reference_flux, + ) + valid_mask[sigma_kept_indices] &= ~raw_ratio_outlier_mask + diagnostics['prefit_raw_ratio_clip_point_count'] = int(np.count_nonzero(valid_mask)) + if diagnostics['prefit_raw_ratio_clip_point_count'] <= 1: + diagnostics.update({ + 'failed_stage': 'prefit_raw_ratio_clip', + 'failure_reason': ( + "pre-fit raw target/reference-ratio outlier clipping left " + f"{diagnostics['prefit_raw_ratio_clip_point_count']} usable point(s); not enough data remained " + "for a lightcurve fit." + ), + }) + return diagnostics + + arrayFinalFlux = flux_ratio_sorted[valid_mask] + f1 = tflux_sorted[valid_mask] + f2 = cflux_sorted[valid_mask] + f1_err = None if target_flux_error_sorted is None else target_flux_error_sorted[valid_mask] + f2_err = None if comp_flux_error_sorted is None else comp_flux_error_sorted[valid_mask] + arrayNormUnc = relative_flux_uncertainty_from_star_errors(f1, f2, f1_err, f2_err) + arrayTimes = times_sorted[valid_mask] + arrayAirmass = airmass_sorted[valid_mask] + + nanmask = np.isnan(arrayFinalFlux) | np.isnan(arrayNormUnc) | np.isnan(arrayTimes) | np.isnan(arrayAirmass) + nanmask = nanmask | np.less_equal(arrayFinalFlux, 0) | np.less_equal(arrayNormUnc, 0) + nanmask = nanmask | np.isinf(arrayFinalFlux) | np.isinf(arrayNormUnc) | np.isinf(arrayTimes) | np.isinf( + arrayAirmass + ) + diagnostics['usable_point_count'] = int(np.count_nonzero(~nanmask)) + if diagnostics['usable_point_count'] <= 1: + diagnostics.update({ + 'failed_stage': 'nan_filter', + 'failure_reason': ( + "filtering non-finite or non-positive flux/uncertainty values left " + f"{diagnostics['usable_point_count']} usable point(s); need at least 2." + ), + }) + elif diagnostics['usable_point_count'] < LIGHTCURVE_MIN_VALID_POINTS: + diagnostics.update({ + 'failed_stage': 'minimum_points', + 'failure_reason': ( + f"only {diagnostics['usable_point_count']} usable point(s) remained after filtering; " + f"need at least {LIGHTCURVE_MIN_VALID_POINTS} for a lightcurve fit." + ), + }) + + return diagnostics + + +def ensure_lightcurve_fit_failure_reason(diagnostics, fit_result, failed_stage, failure_reason): + diagnostics = {} if diagnostics is None else dict(diagnostics) + if fit_result is None and diagnostics.get('failure_reason') is None: + diagnostics.update({ + 'failed_stage': failed_stage, + 'failure_reason': failure_reason, + }) + return diagnostics + + +def prepare_lightcurve_fit_input_series( + times, + target_flux, + comp_flux, + airmass, + target_flux_error=None, + comp_flux_error=None, + jd_times=None, + exposure_times_seconds=None, + gain_e_per_adu=None, + expected_transit_depth=None, +): + times = np.asarray(times, dtype=float) + target_flux = np.asarray(target_flux, dtype=float) + comp_flux = np.asarray(comp_flux, dtype=float) + airmass = np.asarray(airmass, dtype=float) + target_flux_error = valid_flux_error_array(target_flux_error, target_flux.shape) + comp_flux_error = valid_flux_error_array(comp_flux_error, comp_flux.shape) + jd_times_array = None if jd_times is None else np.asarray(jd_times, dtype=float) + exposure_times_array = None if exposure_times_seconds is None else np.asarray(exposure_times_seconds, dtype=float) + + prepared = { + 'applied': False, + 'failure_reason': None, + 'filter_diagnostics': [], + 'debug_times': np.array([], dtype=float), + 'debug_target_flux': np.array([], dtype=float), + 'debug_comp_flux': np.array([], dtype=float), + 'debug_raw_ratio': np.array([], dtype=float), + 'debug_target_flux_error': np.array([], dtype=float), + 'debug_comp_flux_error': np.array([], dtype=float), + 'debug_relative_flux_error': np.array([], dtype=float), + 'initial_sigma_keep_mask': np.array([], dtype=bool), + 'prefit_raw_ratio_keep_mask': np.array([], dtype=bool), + 'time': np.array([], dtype=float), + 'flux': np.array([], dtype=float), + 'unc': np.array([], dtype=float), + 'jd_time': None, + 'exposure_time_seconds': None, + 'airmass': np.array([], dtype=float), + 'target_flux': np.array([], dtype=float), + 'comp_flux': np.array([], dtype=float), + 'target_flux_error': np.array([], dtype=float), + 'comp_flux_error': np.array([], dtype=float), + 'source_indices': np.array([], dtype=int), + 'skip_airmass_fit': False, + 'approximate_baseline_level': np.nan, + } + + if not ( + times.ndim == target_flux.ndim == comp_flux.ndim == airmass.ndim == 1 + and times.shape == target_flux.shape == comp_flux.shape == airmass.shape + ): + prepared['failure_reason'] = ( + "lightcurve inputs must be 1D arrays with matching lengths before fitting." + ) + return prepared + + if jd_times_array is not None and jd_times_array.shape != times.shape: + jd_times_array = None + if exposure_times_array is not None and exposure_times_array.shape != times.shape: + exposure_times_array = None + + target_flux, target_flux_error = scale_target_only_flux_to_common_exposure( + target_flux, + target_flux_error, + comp_flux, + exposure_times_seconds=exposure_times_array, + gain_e_per_adu=gain_e_per_adu, + ) + + plot_indices = np.argsort(times) + times_sorted = times[plot_indices] + target_flux_sorted = target_flux[plot_indices] + comp_flux_sorted = comp_flux[plot_indices] + target_flux_error_sorted = None if target_flux_error is None else target_flux_error[plot_indices] + comp_flux_error_sorted = None if comp_flux_error is None else comp_flux_error[plot_indices] + exposure_times_sorted = None if exposure_times_array is None else exposure_times_array[plot_indices] + source_indices = np.asarray(plot_indices, dtype=int) + with np.errstate(divide='ignore', invalid='ignore'): + flux_ratio_sorted = np.divide(target_flux_sorted, comp_flux_sorted) + + filter_diagnostics = [] + has_reference_flux = not np.allclose(comp_flux_sorted, 1.0) + if has_reference_flux: + flux_ratio_mask = valid_flux_ratio_mask(flux_ratio_sorted) + filter_diagnostics.append(build_time_rejection_diagnostic( + "Target/reference ratio filter", + times_sorted, + flux_ratio_mask, + note="Dropped non-finite or non-positive target/reference ratios before fitting.", + )) + times_sorted = times_sorted[flux_ratio_mask] + target_flux_sorted = target_flux_sorted[flux_ratio_mask] + comp_flux_sorted = comp_flux_sorted[flux_ratio_mask] + if target_flux_error_sorted is not None: + target_flux_error_sorted = target_flux_error_sorted[flux_ratio_mask] + if comp_flux_error_sorted is not None: + comp_flux_error_sorted = comp_flux_error_sorted[flux_ratio_mask] + if exposure_times_sorted is not None: + exposure_times_sorted = exposure_times_sorted[flux_ratio_mask] + flux_ratio_sorted = flux_ratio_sorted[flux_ratio_mask] + source_indices = source_indices[flux_ratio_mask] + if jd_times_array is None: + jd_times_sorted = times_sorted.copy() + else: + jd_times_sorted = jd_times_array[plot_indices][flux_ratio_mask] + airmass_sorted = airmass[plot_indices][flux_ratio_mask] + else: + jd_times_sorted = times_sorted.copy() if jd_times_array is None else jd_times_array[plot_indices] + airmass_sorted = airmass[plot_indices] + + if len(times_sorted) <= 1: + prepared['failure_reason'] = "too few valid points remained after the target/reference ratio filter." + prepared['filter_diagnostics'] = filter_diagnostics + return prepared + + debug_times = np.asarray(times_sorted, dtype=float).copy() + debug_target_flux = np.asarray(target_flux_sorted, dtype=float).copy() + debug_comp_flux = np.asarray(comp_flux_sorted, dtype=float).copy() + debug_raw_ratio = np.asarray(flux_ratio_sorted, dtype=float).copy() + debug_target_flux_error = ( + np.asarray(target_flux_error_sorted, dtype=float).copy() + if target_flux_error_sorted is not None + else np.full(debug_times.shape, np.nan, dtype=float) + ) + debug_comp_flux_error = ( + np.asarray(comp_flux_error_sorted, dtype=float).copy() + if comp_flux_error_sorted is not None + else np.full(debug_times.shape, np.nan, dtype=float) + ) + debug_relative_flux_error = relative_flux_uncertainty_from_star_errors( + target_flux_sorted, + comp_flux_sorted, + target_flux_error_sorted, + comp_flux_error_sorted, + ) + + dt = np.mean(np.diff(times_sorted)) + ndt = int(25. / 24. / 60. / dt) * 2 + 1 + if ndt > len(times_sorted): + ndt = int(len(times_sorted)/4) * 2 + 1 + filtered_data = sigma_clip(flux_ratio_sorted, sigma=3, dt=max(5, ndt), times=times_sorted) + valid_mask = ~filtered_data + initial_sigma_keep_mask = np.asarray(valid_mask, dtype=bool).copy() + filter_diagnostics.append(build_time_rejection_diagnostic( + "Initial sigma clip", + times_sorted, + valid_mask, + note="Dropped 3-sigma target/reference-ratio outliers before the first lightcurve fit.", + )) + + prefit_raw_ratio_keep_mask = np.asarray(initial_sigma_keep_mask, dtype=bool).copy() + sigma_kept_indices = np.flatnonzero(initial_sigma_keep_mask) + if sigma_kept_indices.size: + raw_ratio_outlier_mask = prefit_raw_ratio_outlier_mask( + flux_ratio_sorted[sigma_kept_indices], + times=times_sorted[sigma_kept_indices], + expected_transit_depth=expected_transit_depth, + use_global=has_reference_flux, + ) + raw_ratio_keep_on_sigma_kept = ~np.asarray(raw_ratio_outlier_mask, dtype=bool) + prefit_raw_ratio_keep_mask[sigma_kept_indices] = raw_ratio_keep_on_sigma_kept + filter_diagnostics.append(build_time_rejection_diagnostic( + "Pre-fit raw-ratio outlier clip", + times_sorted[sigma_kept_indices], + raw_ratio_keep_on_sigma_kept, + note=( + "Dropped local log target/reference-ratio outliers before fitting " + "baseline or airmass terms." + ), + )) + else: + filter_diagnostics.append(build_time_rejection_diagnostic( + "Pre-fit raw-ratio outlier clip", + times_sorted, + np.zeros(times_sorted.shape, dtype=bool), + note=( + "Dropped local log target/reference-ratio outliers before fitting " + "baseline or airmass terms." + ), + )) + + valid_mask = prefit_raw_ratio_keep_mask + + flux = flux_ratio_sorted[valid_mask] + filtered_target_flux = target_flux_sorted[valid_mask] + filtered_comp_flux = comp_flux_sorted[valid_mask] + filtered_target_flux_error = None if target_flux_error_sorted is None else target_flux_error_sorted[valid_mask] + filtered_comp_flux_error = None if comp_flux_error_sorted is None else comp_flux_error_sorted[valid_mask] + filtered_exposure_times = None if exposure_times_sorted is None else exposure_times_sorted[valid_mask] + unc = relative_flux_uncertainty_from_star_errors( + filtered_target_flux, + filtered_comp_flux, + filtered_target_flux_error, + filtered_comp_flux_error, + ) + fit_times = times_sorted[valid_mask] + fit_jd_times = jd_times_sorted[valid_mask] + fit_airmass = airmass_sorted[valid_mask] + source_indices = source_indices[valid_mask] + + nanmask = np.isnan(flux) | np.isnan(unc) | np.isnan(fit_times) | np.isnan(fit_airmass) | np.less_equal(flux, 0) | np.less_equal(unc, 0) + nanmask = nanmask | np.isinf(flux) | np.isinf(unc) | np.isinf(fit_times) | np.isinf(fit_airmass) + filter_diagnostics.append(build_time_rejection_diagnostic( + "Finite/positive photometry filter", + fit_times, + ~nanmask, + note="Dropped non-finite or non-positive flux, uncertainty, time, or airmass values.", + )) + + if np.sum(~nanmask) <= 1: + prepared['failure_reason'] = "too few valid points remained after removing non-finite or non-positive photometry." + prepared.update({ + 'filter_diagnostics': filter_diagnostics, + 'debug_times': debug_times, + 'debug_target_flux': debug_target_flux, + 'debug_comp_flux': debug_comp_flux, + 'debug_raw_ratio': debug_raw_ratio, + 'debug_target_flux_error': debug_target_flux_error, + 'debug_comp_flux_error': debug_comp_flux_error, + 'debug_relative_flux_error': debug_relative_flux_error, + 'initial_sigma_keep_mask': initial_sigma_keep_mask, + 'prefit_raw_ratio_keep_mask': prefit_raw_ratio_keep_mask, + }) + return prepared + + normalized_flux, normalized_unc, approximate_baseline_level = normalize_flux_series_to_approximate_unity( + flux[~nanmask], + unc[~nanmask], + ) + + prepared.update({ + 'applied': True, + 'filter_diagnostics': filter_diagnostics, + 'debug_times': debug_times, + 'debug_target_flux': debug_target_flux, + 'debug_comp_flux': debug_comp_flux, + 'debug_raw_ratio': debug_raw_ratio, + 'debug_target_flux_error': debug_target_flux_error, + 'debug_comp_flux_error': debug_comp_flux_error, + 'debug_relative_flux_error': debug_relative_flux_error, + 'initial_sigma_keep_mask': initial_sigma_keep_mask, + 'prefit_raw_ratio_keep_mask': prefit_raw_ratio_keep_mask, + 'time': fit_times[~nanmask], + 'flux': normalized_flux, + 'unc': normalized_unc, + 'jd_time': fit_jd_times[~nanmask], + 'exposure_time_seconds': None if filtered_exposure_times is None else filtered_exposure_times[~nanmask], + 'airmass': fit_airmass[~nanmask], + 'target_flux': filtered_target_flux[~nanmask], + 'comp_flux': filtered_comp_flux[~nanmask], + 'target_flux_error': np.full(filtered_target_flux.shape, np.nan, dtype=float)[~nanmask] + if filtered_target_flux_error is None else filtered_target_flux_error[~nanmask], + 'comp_flux_error': np.full(filtered_comp_flux.shape, np.nan, dtype=float)[~nanmask] + if filtered_comp_flux_error is None else filtered_comp_flux_error[~nanmask], + 'source_indices': source_indices[~nanmask], + 'skip_airmass_fit': should_skip_airmass_fit(fit_airmass[~nanmask]), + 'approximate_baseline_level': approximate_baseline_level, + }) + return prepared + + +def comparison_prescore_clip_mask(values, sigma=COMPARISON_STAR_PRESCORE_SIGMA_CLIP, + max_iters=COMPARISON_STAR_PRESCORE_MAX_CLIP_ITERS, + scatter_floor=COMPARISON_STAR_PRESCORE_SCATTER_FLOOR, + min_points=LIGHTCURVE_MIN_VALID_POINTS): + values = np.asarray(values, dtype=float) + finite_mask = np.isfinite(values) + keep_mask = finite_mask.copy() + if np.count_nonzero(keep_mask) < int(min_points): + return keep_mask + + for _ in range(int(max_iters)): + kept_values = values[keep_mask] + center = bn.nanmedian(kept_values) + if not np.isfinite(center): + break + + mad = bn.nanmedian(np.abs(kept_values - center)) + if np.isfinite(mad) and mad > 0: + scatter = 1.4826 * mad + else: + scatter = bn.nanstd(kept_values) + + if np.isfinite(scatter_floor) and scatter_floor > 0: + if not np.isfinite(scatter) or scatter <= 0: + scatter = float(scatter_floor) else: - log_info(f"\n {pdict['pName']} {planet_params[i]}: {pdict[key]}") - agreement = user_input("Do you agree? (y/n): ", type_=str, values=['y', 'n']) - if agreement == 'y': - userpdict[key] = pdict[key] - else: - userpdict[key] = user_input(f"Enter the {planet_params[i]}: ", type_=type(pdict[key])) + scatter = max(float(scatter), float(scatter_floor)) + + if not np.isfinite(scatter) or scatter <= 0: + break + + next_keep_mask = finite_mask & (np.abs(values - center) <= float(sigma) * scatter) + if np.count_nonzero(next_keep_mask) < int(min_points): + break + if np.array_equal(next_keep_mask, keep_mask): + break + keep_mask = next_keep_mask + + return keep_mask + + +def cheap_lightcurve_prescore(tFlux, cFlux, airmass, enforce_relative_flux_max=True): + with np.errstate(divide='ignore', invalid='ignore'): + flux_ratio = np.divide(tFlux, cFlux) + + finite_mask = valid_flux_ratio_mask(flux_ratio) & np.isfinite(airmass) + if np.count_nonzero(finite_mask) < 5: + return np.inf + + x_vals = airmass[finite_mask] + y_vals = flux_ratio[finite_mask] + + score_mask = np.ones(y_vals.shape, dtype=bool) + detrended = np.full(y_vals.shape, np.nan, dtype=float) + for _ in range(COMPARISON_STAR_PRESCORE_MAX_CLIP_ITERS): + if np.count_nonzero(score_mask) < LIGHTCURVE_MIN_VALID_POINTS: + return np.inf + + if should_skip_airmass_fit(x_vals[score_mask]): + baseline = bn.nanmedian(y_vals[score_mask]) + if not np.isfinite(baseline) or baseline == 0: + return np.inf + detrended = y_vals / baseline + else: + slope, intercept = np.polyfit(x_vals[score_mask], y_vals[score_mask], 1) + trend = slope * x_vals + intercept + with np.errstate(divide='ignore', invalid='ignore'): + detrended = np.divide(y_vals, trend) + + next_score_mask = comparison_prescore_clip_mask(detrended) + if np.array_equal(next_score_mask, score_mask): + break + score_mask = next_score_mask + + finite_detrended = detrended[score_mask & np.isfinite(detrended)] + if finite_detrended.size < LIGHTCURVE_MIN_VALID_POINTS: + return np.inf + + scatter = bn.nanstd(finite_detrended) + if np.isfinite(scatter) and scatter >= 0: + return float(scatter) + + return np.inf + + +def target_comp_flux_scatter(tFlux, cFlux, min_points=LIGHTCURVE_MIN_VALID_POINTS): + if tFlux is None or cFlux is None: + return np.nan + + tFlux = np.asarray(tFlux, dtype=float) + cFlux = np.asarray(cFlux, dtype=float) + if tFlux.shape != cFlux.shape: + return np.nan + + with np.errstate(divide='ignore', invalid='ignore'): + flux_ratio = np.divide(tFlux, cFlux) + + ratio_mask = valid_flux_ratio_mask(flux_ratio) + if np.count_nonzero(ratio_mask) < int(min_points): + return np.nan + + ratio_values = np.asarray(flux_ratio[ratio_mask], dtype=float) + baseline = bn.nanmedian(ratio_values) + if not np.isfinite(baseline) or baseline <= 0: + return np.nan + + normalized_ratio = ratio_values / baseline + keep_mask = comparison_prescore_clip_mask( + normalized_ratio, + min_points=min_points, + ) + kept_ratio = normalized_ratio[keep_mask & np.isfinite(normalized_ratio)] + if kept_ratio.size < int(min_points): + return np.nan + + scatter = robust_scatter(kept_ratio - bn.nanmedian(kept_ratio)) + return float(scatter) if np.isfinite(scatter) and scatter >= 0 else np.nan + + +def fitted_lightcurve_model_at(fit, times, airmass): + if fit is None: + return np.array([], dtype=float) + + parameters = getattr(fit, 'parameters', None) + if not isinstance(parameters, dict): + return np.array([], dtype=float) + + times = np.asarray(times, dtype=float) + airmass = np.asarray(airmass, dtype=float) + if times.ndim != 1 or airmass.shape != times.shape: + return np.array([], dtype=float) + + try: + fit_transit_model = getattr(fit, '_transit_model', None) + if callable(fit_transit_model): + transit_model = np.asarray(fit_transit_model(times, parameters), dtype=float) + else: + transit_model = np.asarray(transit(times, parameters), dtype=float) + except Exception: + return np.array([], dtype=float) + if transit_model.shape != times.shape: + return np.array([], dtype=float) + + baseline = parameters.get('a0', parameters.get('a1', 1.0)) + try: + baseline = float(baseline) + except (TypeError, ValueError): + baseline = 1.0 + if not np.isfinite(baseline): + baseline = 1.0 + + try: + a2 = float(parameters.get('a2', 0.0)) + except (TypeError, ValueError): + a2 = 0.0 + if not np.isfinite(a2): + a2 = 0.0 + + reference = getattr(fit, 'airmass_reference', None) + try: + reference = float(reference) + except (TypeError, ValueError): + reference = transit_qc_airmass_reference(getattr(fit, 'airmass', airmass)) + if not np.isfinite(reference): + reference = transit_qc_airmass_reference(airmass) + + systematics = baseline * transit_qc_airmass_trend(a2, airmass, reference=reference) + model = transit_model * systematics + return model if model.shape == times.shape else np.array([], dtype=float) + + +def fitted_lightcurve_scatter_on_dataset(fit, times, flux_values, airmass): + times = np.asarray(times, dtype=float) + flux_values = np.asarray(flux_values, dtype=float) + airmass = np.asarray(airmass, dtype=float) + if not (times.shape == flux_values.shape == airmass.shape): + return np.nan + + model = fitted_lightcurve_model_at(fit, times, airmass) + if model.shape != flux_values.shape: + return np.nan + + scatter = transit_qc_residual_scatter(flux_values, model) + return float(scatter) if np.isfinite(scatter) and scatter >= 0 else np.nan + + +def evaluate_lightcurve_candidate(task): + exposure_times_seconds = None + gain_e_per_adu = None + if len(task) == 14: + ( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times, + plot_time_range, + disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit, + use_eebls_to_initialize_tmid_and_bounds, + compute_eebls_diagnostics, + exposure_times_seconds, + gain_e_per_adu, + ) = task + elif len(task) == 13: + ( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times, + plot_time_range, + disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit, + use_eebls_to_initialize_tmid_and_bounds, + compute_eebls_diagnostics, + exposure_times_seconds, + ) = task + else: + ( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times, + plot_time_range, + disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit, + use_eebls_to_initialize_tmid_and_bounds, + compute_eebls_diagnostics, + ) = task + fit_diagnostics = diagnose_lightcurve_fit_inputs( + times, + tflux, + cflux, + airmass, + enforce_relative_flux_max=False, + expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), + ) + myfit, tflux_fit, cflux_fit = fit_lightcurve( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times, + allow_mid_transit_range_warning=False, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + final_fit_mode='ns', + use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + plot_time_range=plot_time_range, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, + compute_eebls_diagnostics=compute_eebls_diagnostics, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=gain_e_per_adu, + ) + fit_diagnostics = ensure_lightcurve_fit_failure_reason( + fit_diagnostics, + myfit, + failed_stage='lightcurve_fit', + failure_reason="the lightcurve fitter did not converge to a usable solution.", + ) + transit_qc_failure_reason = lightcurve_fit_transit_qc_failure_reason(myfit) + if transit_qc_failure_reason is not None: + fit_diagnostics = dict(fit_diagnostics) + fit_diagnostics.update({ + 'failed_stage': 'transit_qc', + 'failure_reason': transit_qc_failure_reason, + }) + + return { + 'myfit': myfit, + 'accepted': myfit is not None and transit_qc_failure_reason is None, + 'eebls_snr': extract_lightcurve_fit_eebls_snr(myfit), + 'transit_delta_bic': extract_lightcurve_fit_transit_delta_bic(myfit), + 'residual_scatter': extract_lightcurve_fit_residual_scatter(myfit), + 'ktmf_metric': extract_lightcurve_fit_ktmf_metric(myfit), + 'ktmf_contributions': extract_lightcurve_fit_ktmf_contributions(myfit), + 'transit_qc_status': getattr(myfit, 'transit_qc_status', None), + 'transit_qc_summary': getattr(myfit, 'transit_qc_summary', None), + 'rejected_by_transit_qc': transit_qc_failure_reason is not None, + 'fit_diagnostics': fit_diagnostics, + 'failure_reason': fit_diagnostics.get('failure_reason'), + 'fit_point_count': 0 if tflux_fit is None else int(len(tflux_fit)), + }, tflux_fit, cflux_fit + + +def build_target_fit_candidate_jobs(psf_data, aper_data, apers, annuli, airmass, comp_stars, sigma, + require_comp_star=True, + skip_low_comparison_coverage_rejection=False, + use_psf_photometry=True, + use_aperture_photometry=True, + psf_flux_data=None): + candidate_jobs = [] + comp_star_count = len(comp_stars) + psf_flux_data = psf_flux_data_source(psf_data, psf_flux_data) + + if use_psf_photometry and comp_star_count > 0: + frame_count = psf_data['target'].shape[0] + target_flux = psf_flux_series_from_rows(psf_flux_data['target']) + target_flux_mask = robust_flux_floor_mask(target_flux) + psf_comp_flux_map = { + f"comp{comp_idx + 1}": psf_flux_series_from_rows( + psf_flux_data[f"comp{comp_idx + 1}"], + psf_quality_mask_for_key( + psf_data, + f"comp{comp_idx + 1}", + frame_count, + psf_flux_data=psf_flux_data, + ), + ) + for comp_idx in range(comp_star_count) + } + psf_comp_coverage = comparison_star_coverage_summary( + psf_comp_flux_map, + skip_rejection=skip_low_comparison_coverage_rejection, + validity_mask_func=robust_flux_floor_mask, + ) + for comp_idx in range(comp_star_count): + ckey = f"comp{comp_idx + 1}" + if psf_comp_coverage[ckey]['coverage_rejected']: + continue - # Exoplanet not confirmed in NASA Exoplanet Archive + comp_flux = psf_comp_flux_map[ckey] + target_shape_mask = target_psf_shape_quality_mask( + target_psf_quality_rows(psf_data, psf_flux_data=psf_flux_data), + psf_quality_rows_for_key(psf_data, ckey, psf_flux_data=psf_flux_data), + ) + psf_mask = target_shape_mask & target_flux_mask & robust_flux_floor_mask(comp_flux) + candidate_jobs.append({ + 'method': 'psf', + 'a': None, + 'an': None, + 'aper': 0.0, + 'annulus': float(15 * sigma), + 'comp_index': comp_idx, + 'ckey': ckey, + 'mask': psf_mask, + 'coverage_count': psf_comp_coverage[ckey]['coverage_count'], + 'coverage_total_frame_count': psf_comp_coverage[ckey]['coverage_total_frame_count'], + 'coverage_reference_count': psf_comp_coverage[ckey]['coverage_reference_count'], + 'coverage_min_required_count': psf_comp_coverage[ckey]['coverage_min_required_count'], + 'coverage_rejected': psf_comp_coverage[ckey]['coverage_rejected'], + 'prescore': cheap_lightcurve_prescore( + target_flux[psf_mask], + comp_flux[psf_mask], + airmass[psf_mask], + enforce_relative_flux_max=False, + ), + }) + + if use_aperture_photometry and aper_data is not None and apers is not None and annuli is not None: + frame_count = aper_data['target'].shape[0] + for a, aper in enumerate(apers): + for an, annulus in enumerate(annuli): + target_flux = np.asarray(aper_data['target'][:, a, an], dtype=float) + aperture_comp_flux_map = { + f"comp{comp_idx + 1}": mask_series_with_quality( + aper_data[f"comp{comp_idx + 1}"][:, a, an], + psf_quality_mask_for_key(psf_data, f"comp{comp_idx + 1}", frame_count), + ) + for comp_idx in range(comp_star_count) + } + aperture_comp_coverage = comparison_star_coverage_summary( + aperture_comp_flux_map, + skip_rejection=skip_low_comparison_coverage_rejection, + ) + + if not require_comp_star: + candidate_jobs.append({ + 'method': 'aperture', + 'a': a, + 'an': an, + 'aper': float(aper), + 'annulus': float(annulus), + 'comp_index': None, + 'ckey': None, + 'mask': np.ones(target_flux.shape[0], dtype=bool), + 'coverage_count': int(target_flux.shape[0]), + 'coverage_total_frame_count': int(target_flux.shape[0]), + 'coverage_reference_count': float(target_flux.shape[0]), + 'coverage_min_required_count': LIGHTCURVE_MIN_VALID_POINTS, + 'coverage_rejected': False, + 'prescore': cheap_lightcurve_prescore( + target_flux, + np.ones(target_flux.shape[0]), + airmass, + enforce_relative_flux_max=False, + ), + }) + + for comp_idx in range(comp_star_count): + ckey = f"comp{comp_idx + 1}" + if aperture_comp_coverage[ckey]['coverage_rejected']: + continue + + comp_series = aperture_comp_flux_map[ckey] + aper_mask = valid_comparison_frame_mask(target_flux) & valid_comparison_frame_mask(comp_series) + candidate_jobs.append({ + 'method': 'aperture', + 'a': a, + 'an': an, + 'aper': float(aper), + 'annulus': float(annulus), + 'comp_index': comp_idx, + 'ckey': ckey, + 'mask': aper_mask, + 'coverage_count': aperture_comp_coverage[ckey]['coverage_count'], + 'coverage_total_frame_count': aperture_comp_coverage[ckey]['coverage_total_frame_count'], + 'coverage_reference_count': aperture_comp_coverage[ckey]['coverage_reference_count'], + 'coverage_min_required_count': aperture_comp_coverage[ckey]['coverage_min_required_count'], + 'coverage_rejected': aperture_comp_coverage[ckey]['coverage_rejected'], + 'prescore': cheap_lightcurve_prescore( + target_flux[aper_mask], + comp_series[aper_mask], + airmass[aper_mask], + enforce_relative_flux_max=False, + ), + }) + + return candidate_jobs + + +def target_fit_candidate_task(candidate, times, jd_times, airmass, ld, p_dict, psf_data, aper_data, + plot_time_range=None, + disable_vertical_flux_normalization=False, + use_impactparameter_rather_than_inclination_to_fit=True, + use_eebls_to_initialize_tmid_and_bounds=True, + compute_eebls_diagnostics=True, + exposure_times_seconds=None, + gain_e_per_adu=None, + psf_flux_data=None): + candidate_mask = np.asarray(candidate['mask'], dtype=bool) + + if candidate['method'] == 'psf': + psf_flux_data = psf_flux_data_source(psf_data, psf_flux_data) + target_flux = ( + 2 * np.pi * psf_flux_data['target'][:, 2] + * psf_flux_data['target'][:, 3] + * psf_flux_data['target'][:, 4] + ) + if candidate['ckey'] is None: + comp_flux = np.ones(target_flux.shape[0], dtype=float) + else: + comp_flux = ( + 2 * np.pi * psf_flux_data[candidate['ckey']][:, 2] + * psf_flux_data[candidate['ckey']][:, 3] + * psf_flux_data[candidate['ckey']][:, 4] + ) else: - for i, key in enumerate(userpdict): - if key in ('ra', 'dec'): - continue - # Used initialization file and is not empty - if userpdict[key] is not None: - agreement = user_input(f"{planet_params[i]}: {userpdict[key]} \nDo you agree? (y/n): ", - type_=str, values=['y', 'n']) - if agreement == 'y': - continue - else: - userpdict[key] = user_input(f"Enter the {planet_params[i]}: ", type_=type(userpdict[key])) - # Did not use initialization file + target_flux = aper_data['target'][:, candidate['a'], candidate['an']] + if candidate['ckey'] is None: + comp_flux = np.ones(target_flux.shape[0], dtype=float) + else: + comp_flux = aper_data[candidate['ckey']][:, candidate['a'], candidate['an']] + + return ( + times[candidate_mask], + target_flux[candidate_mask], + comp_flux[candidate_mask], + airmass[candidate_mask], + ld, + p_dict, + jd_times[candidate_mask], + plot_time_range, + disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit, + use_eebls_to_initialize_tmid_and_bounds, + compute_eebls_diagnostics, + None if exposure_times_seconds is None else np.asarray(exposure_times_seconds, dtype=float)[candidate_mask], + gain_e_per_adu, + ) + + +def run_target_driven_photometry_search(times, jd_times, airmass, ld, p_dict, comp_stars, psf_data, aper_data, + apers, annuli, sigma, + require_comp_star=True, + plot_time_range=None, + disable_vertical_flux_normalization=False, + skip_low_comparison_coverage_rejection=False, + use_psf_photometry=True, + use_aperture_photometry=True, + multiprocess_lightcurve_fits=None, + use_impactparameter_rather_than_inclination_to_fit=True, + use_eebls_to_initialize_tmid_and_bounds=True, + pick_comparison_by_eebls_snr=True, + exposure_times_seconds=None, + gain_e_per_adu=None, + psf_flux_data=None): + candidate_jobs = build_target_fit_candidate_jobs( + psf_data, + aper_data, + apers, + annuli, + airmass, + comp_stars, + sigma, + require_comp_star=require_comp_star, + skip_low_comparison_coverage_rejection=skip_low_comparison_coverage_rejection, + use_psf_photometry=use_psf_photometry, + use_aperture_photometry=use_aperture_photometry, + psf_flux_data=psf_flux_data, + ) + evaluated_candidates = list(candidate_jobs) + for candidate_order, candidate in enumerate(evaluated_candidates): + candidate.setdefault('candidate_order', candidate_order) + if not evaluated_candidates: + return { + 'candidate_jobs': candidate_jobs, + 'evaluated_candidates': evaluated_candidates, + 'shortlist': evaluated_candidates, + 'candidate_summaries': [], + 'best_candidate': None, + 'best_fit_lc': None, + 'selected_ktmf_metric': np.nan, + 'selected_transit_delta_bic': np.nan, + 'selection_metric': 'ktmf', + 'selected_eebls_snr': np.nan, + 'flux_tar': None, + 'flux_ref': None, + 'selected_source_indices': None, + } + + fit_tasks = [ + target_fit_candidate_task( + candidate, + times, + jd_times, + airmass, + ld, + p_dict, + psf_data, + aper_data, + plot_time_range=plot_time_range, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, + compute_eebls_diagnostics=True, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=gain_e_per_adu, + psf_flux_data=psf_flux_data, + ) + for candidate in evaluated_candidates + ] + + if multiprocess_lightcurve_fits is not None and multiprocess_lightcurve_fits > 0: + log_info(f"Using multiprocessing for candidate lightcurve fits ({multiprocess_lightcurve_fits} processes).") + with suppress_tk_cleanup_during_process_pool(): + with ProcessPoolExecutor( + max_workers=multiprocess_lightcurve_fits, + initializer=suppress_inherited_tk_cleanup_in_worker, + ) as executor: + fit_results = list(executor.map(evaluate_lightcurve_candidate, fit_tasks)) + else: + fit_results = [evaluate_lightcurve_candidate(task) for task in fit_tasks] + + candidate_summaries = [] + successful_candidates = [] + for candidate, result in zip(evaluated_candidates, fit_results): + fit_meta, tflux_fit, cflux_fit = result + summary = summarize_target_fit_candidate(candidate, fit_meta, comp_stars) + candidate_summaries.append(summary) + + if fit_meta is None or fit_meta.get('myfit') is None or not fit_meta.get('accepted', True): + continue + successful_candidates.append((summary, candidate, fit_meta, tflux_fit, cflux_fit)) + + best_candidate = None + best_fit_lc = None + best_ktmf_metric = np.nan + best_transit_delta_bic = np.nan + best_tflux = None + best_cflux = None + selection_metric = 'ktmf' + selected_eebls_snr = np.nan + selected_source_indices = None + if successful_candidates: + has_ktmf = any(np.isfinite(item[0].get('ktmf_metric', np.nan)) for item in successful_candidates) + has_eebls = any(np.isfinite(item[0].get('eebls_snr', np.nan)) for item in successful_candidates) + has_delta_bic = any(np.isfinite(item[0].get('transit_delta_bic', np.nan)) for item in successful_candidates) + + if has_ktmf: + selection_metric = 'ktmf' + if pick_comparison_by_eebls_snr: + selected_entry = min( + successful_candidates, + key=lambda item: ( + 0 if np.isfinite(item[0].get('ktmf_metric', np.nan)) else 1, + -item[0].get('ktmf_metric', np.nan) if np.isfinite(item[0].get('ktmf_metric', np.nan)) else np.inf, + 0 if np.isfinite(item[0].get('eebls_snr', np.nan)) else 1, + -item[0].get('eebls_snr', np.nan) if np.isfinite(item[0].get('eebls_snr', np.nan)) else np.inf, + 0 if np.isfinite(item[0].get('transit_delta_bic', np.nan)) else 1, + -item[0].get('transit_delta_bic', np.nan) if np.isfinite(item[0].get('transit_delta_bic', np.nan)) else np.inf, + item[0].get('candidate_order', np.inf), + ), + ) else: - if key in ('pName', 'sName'): - userpdict[key] = user_input(f"\nEnter the {planet_params[i]}: ", type_=str) - else: - userpdict[key] = user_input(f"Enter the {planet_params[i]}: ", type_=float) - return userpdict + selected_entry = min( + successful_candidates, + key=lambda item: ( + 0 if np.isfinite(item[0].get('ktmf_metric', np.nan)) else 1, + -item[0].get('ktmf_metric', np.nan) if np.isfinite(item[0].get('ktmf_metric', np.nan)) else np.inf, + 0 if np.isfinite(item[0].get('transit_delta_bic', np.nan)) else 1, + -item[0].get('transit_delta_bic', np.nan) if np.isfinite(item[0].get('transit_delta_bic', np.nan)) else np.inf, + 0 if np.isfinite(item[0].get('eebls_snr', np.nan)) else 1, + -item[0].get('eebls_snr', np.nan) if np.isfinite(item[0].get('eebls_snr', np.nan)) else np.inf, + item[0].get('candidate_order', np.inf), + ), + ) + elif pick_comparison_by_eebls_snr and has_eebls: + selection_metric = 'eebls_snr' + selected_entry = min( + successful_candidates, + key=lambda item: ( + 0 if np.isfinite(item[0].get('eebls_snr', np.nan)) else 1, + -item[0].get('eebls_snr', np.nan) if np.isfinite(item[0].get('eebls_snr', np.nan)) else np.inf, + 0 if np.isfinite(item[0].get('transit_delta_bic', np.nan)) else 1, + -item[0].get('transit_delta_bic', np.nan) if np.isfinite(item[0].get('transit_delta_bic', np.nan)) else np.inf, + item[0].get('candidate_order', np.inf), + ), + ) + elif has_delta_bic: + selection_metric = 'transit_delta_bic' + selected_entry = min( + successful_candidates, + key=lambda item: ( + 0 if np.isfinite(item[0].get('transit_delta_bic', np.nan)) else 1, + -item[0].get('transit_delta_bic', np.nan) if np.isfinite(item[0].get('transit_delta_bic', np.nan)) else np.inf, + 0 if np.isfinite(item[0].get('eebls_snr', np.nan)) else 1, + -item[0].get('eebls_snr', np.nan) if np.isfinite(item[0].get('eebls_snr', np.nan)) else np.inf, + item[0].get('candidate_order', np.inf), + ), + ) + elif has_eebls: + selection_metric = 'eebls_snr' + selected_entry = min( + successful_candidates, + key=lambda item: ( + 0 if np.isfinite(item[0].get('eebls_snr', np.nan)) else 1, + -item[0].get('eebls_snr', np.nan) if np.isfinite(item[0].get('eebls_snr', np.nan)) else np.inf, + item[0].get('candidate_order', np.inf), + ), + ) + else: + selection_metric = 'comparison_field_rank' + selected_entry = min( + successful_candidates, + key=lambda item: item[0].get('candidate_order', np.inf), + ) + selected_summary, best_candidate, fit_meta, best_tflux, best_cflux = selected_entry + best_fit_lc = fit_meta['myfit'] + best_ktmf_metric = selected_summary.get('ktmf_metric', np.nan) + best_transit_delta_bic = selected_summary.get('transit_delta_bic', np.nan) + selected_eebls_snr = selected_summary.get('eebls_snr', np.nan) + candidate_source_indices = np.flatnonzero(np.asarray(best_candidate['mask'], dtype=bool)) + fitted_times = np.asarray(getattr(best_fit_lc, 'time', []), dtype=float) + candidate_times = np.asarray(times, dtype=float)[candidate_source_indices] + fitted_time_indices = ( + match_time_subset_indices(candidate_times, fitted_times) + if fitted_times.size + else None + ) + if fitted_time_indices is not None: + selected_source_indices = candidate_source_indices[fitted_time_indices] + elif best_tflux is not None and len(best_tflux) == len(candidate_source_indices): + selected_source_indices = candidate_source_indices + best_identity = target_fit_candidate_identity(best_candidate) + for summary in candidate_summaries: + summary['selected'] = target_fit_candidate_identity(summary) == best_identity + + return { + 'candidate_jobs': candidate_jobs, + 'evaluated_candidates': evaluated_candidates, + 'shortlist': evaluated_candidates, + 'candidate_summaries': candidate_summaries, + 'best_candidate': best_candidate, + 'best_fit_lc': best_fit_lc, + 'selected_ktmf_metric': best_ktmf_metric, + 'selected_transit_delta_bic': best_transit_delta_bic, + 'selection_metric': selection_metric, + 'selected_eebls_snr': selected_eebls_snr, + 'flux_tar': best_tflux, + 'flux_ref': best_cflux, + 'selected_source_indices': selected_source_indices, + } -# Conversion of Right Ascension and Declination: hours -> degrees -def radec_hours_to_degree(ra, dec): - while True: + +def apply_raw_target_photometry_selection(target_driven_search, photometry_info, flux_values, + centroid_positions, psf_data): + best_candidate = target_driven_search.get('best_candidate') + best_fit_lc = target_driven_search.get('best_fit_lc') + if ( + best_candidate is None + or best_fit_lc is None + or best_candidate.get('comp_index') is not None + or best_candidate.get('method') != 'aperture' + ): + return False + + aperture = float(best_candidate.get('aper', np.nan)) + annulus = float(best_candidate.get('annulus', np.nan)) + if not np.isfinite(aperture) or aperture <= 0 or not np.isfinite(annulus): + return False + + target_flux = np.asarray(target_driven_search.get('flux_tar'), dtype=float) + reference_flux = np.asarray(target_driven_search.get('flux_ref'), dtype=float) + if target_flux.ndim != 1 or reference_flux.shape != target_flux.shape or target_flux.size == 0: + return False + + source_indices = target_driven_search.get('selected_source_indices') + if source_indices is None: + source_indices = np.arange(target_flux.size, dtype=int) + source_indices = np.asarray(source_indices, dtype=int) + target_rows = np.asarray(psf_data.get('target', [])) + if ( + source_indices.shape != target_flux.shape + or target_rows.ndim < 2 + or target_rows.shape[1] < 2 + or np.any(source_indices < 0) + or np.any(source_indices >= target_rows.shape[0]) + ): + return False + + selected_summary = next( + ( + summary for summary in target_driven_search.get('candidate_summaries', []) + if summary.get('selected') + ), + {}, + ) + photometry_info.update( + best_fit_lc=best_fit_lc, + comp_star_num=None, + comp_star_coords=None, + finder_comparison_entries=[], + min_aperture=-abs(aperture), + min_annulus=annulus, + aperture_index=best_candidate.get('a'), + annulus_index=best_candidate.get('an'), + selected_source_indices=source_indices, + selection_basis='raw_target_flux_fallback', + selection_metric=target_driven_search.get('selection_metric', 'ktmf'), + comparison_ktmf_metric=target_driven_search.get('selected_ktmf_metric', np.nan), + comparison_eebls_snr=target_driven_search.get('selected_eebls_snr', np.nan), + comparison_transit_delta_bic=target_driven_search.get('selected_transit_delta_bic', np.nan), + selected_comparison_fit_point_count=selected_summary.get('fit_point_count'), + selected_comparison_transit_qc_status=selected_summary.get('transit_qc_status'), + selected_comparison_transit_qc_summary=selected_summary.get('transit_qc_summary'), + ) + target_uncertainty = np.sqrt(np.clip(target_flux, 0.0, None)) + flux_values.update( + flux_tar=target_flux, + flux_ref=reference_flux, + flux_unc_tar=target_uncertainty, + flux_unc_ref=np.zeros(reference_flux.shape, dtype=float), + ) + target_x = target_rows[source_indices, 0] + target_y = target_rows[source_indices, 1] + centroid_positions.update( + x_targ=target_x, + y_targ=target_y, + x_ref=np.full(target_x.shape, np.nan, dtype=float), + y_ref=np.full(target_y.shape, np.nan, dtype=float), + ) + return True + + +def selected_photometry_method_label(photometry_info): + min_aperture = photometry_info.get('min_aperture') + min_annulus = photometry_info.get('min_annulus') + + if min_aperture == 0: + return "PSF photometry" + if min_aperture is None: + return "Photometry" + + aper_text = abs(float(min_aperture)) + if min_annulus is None or not np.isfinite(min_annulus): + return f"Aperture photometry (aper={aper_text:.2f}px)" + return f"Aperture photometry (aper={aper_text:.2f}px, annulus={float(min_annulus):.2f}px)" + + +def format_comp_star_position(position): + if position is None: + return "x=n/a, y=n/a" + + try: + x_pos, y_pos = position + return f"x={float(x_pos):.1f}, y={float(y_pos):.1f}" + except (TypeError, ValueError): + return f"coords={position}" + + +def deduplicate_comparison_star_coords(comp_stars, min_separation_pixels=COMPARISON_STAR_DUPLICATE_DISTANCE_PIXELS): + """Normalize comparison-star coordinates without merging nearby stars. + + The function name is historical. User-provided comparison-star selections + are intentional inputs, so nearby stars must remain distinct candidates. + """ + if comp_stars is None: + return [], [] + + normalized_coords = [] + for coord in comp_stars: try: - ra = ra.replace(' ', '').replace(':', ' ') - dec = dec.replace(' ', '').replace(':', ' ') - c = SkyCoord(ra + ' ' + dec, unit=(u.hourangle, u.deg)) - return c.ra.degree, c.dec.degree - except ValueError: - log_info("Error: The format entered for Right Ascension and/or Declination is not correct, " - "please try again.", error=True) - ra = input("Input the Right Ascension of target (HH:MM:SS): ") - dec = input("Input the Declination of target (DD:MM:SS): ") + x_pos, y_pos = float(coord[0]), float(coord[1]) + except (TypeError, ValueError, IndexError): + continue + normalized_coords.append([x_pos, y_pos]) -def check_all_standard_filters(ld, observed_filter): - if ld.check_standard(observed_filter): - return True - elif observed_filter['filter']: - filter_name = observed_filter['filter'].lower().replace(' ', '') - filter_name = re.sub(ld_re_punct_p, '', filter_name) - filter_abbreviation = next((filter_abbr for filter_abbr in LimbDarkening.fwhm_names_nonspecific.keys() - if filter_name == filter_abbr.lower()), None) - filter_desc = next((filter_desc for filter_desc in LimbDarkening.fwhm_names_nonspecific.values() - if filter_name == re.sub(ld_re_punct_p, '', filter_desc.lower().replace(' ', ''))), - None) + return normalized_coords, [] - if filter_abbreviation: - observed_filter['filter'] = LimbDarkening.fwhm_names_nonspecific.get(filter_abbreviation) - observed_filter['name'] = filter_abbreviation - custom_range(ld, observed_filter) - return True - if filter_desc: - observed_filter['filter'] = filter_desc - observed_filter['name'] = next((k for k, v in LimbDarkening.fwhm_names_nonspecific.items() if v == filter_desc)) - custom_range(ld, observed_filter) - return True +def merge_automatic_comparison_star_coords(primary_stars, additional_stars, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS): + """Append automatic candidates without duplicating an already tracked sky source. - return False + Primary coordinates are preserved exactly because they can be intentional user + selections. The proximity rule only governs automatic additions to that list. + """ + merged, _ = deduplicate_comparison_star_coords(primary_stars) + messages = [] + try: + duplicate_radius = max(float(duplicate_radius_pixels), 0.0) + except (TypeError, ValueError): + duplicate_radius = REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS + for coord in additional_stars or []: + try: + candidate = [float(coord[0]), float(coord[1])] + except (TypeError, ValueError, IndexError): + continue + duplicate_index = next(( + index for index, existing in enumerate(merged) + if np.hypot(candidate[0] - existing[0], candidate[1] - existing[1]) <= duplicate_radius + ), None) + if duplicate_index is not None: + messages.append( + "Skipped automatic comparison candidate " + f"[{candidate[0]:.2f}, {candidate[1]:.2f}] because it duplicates tracked " + f"comparison star #{duplicate_index + 1} within {duplicate_radius:.1f} pixels." + ) + continue + merged.append(candidate) + return merged, messages + + +def build_tracked_comparison_pool(science_comp_stars, automatic_comp_stars, + use_exactly_the_comps_provided=False, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS): + """Build the tracked comparison pool without expanding an exact supplied set.""" + supplied_stars = [list(position) for position in (science_comp_stars or [])] + if use_exactly_the_comps_provided: + return supplied_stars, [] + return merge_automatic_comparison_star_coords( + supplied_stars, + automatic_comp_stars, + duplicate_radius_pixels=duplicate_radius_pixels, + ) -def custom_range(ld, observed_filter): - while True: - if ld.check_fwhm(observed_filter): - ld.set_filter(observed_filter['name'], observed_filter['filter'], - float(observed_filter['wl_min']), float(observed_filter['wl_max'])) - return - else: - observed_filter['wl_min'] = user_input(f"FWHM minimum wavelength (nm):", type_=str) - observed_filter['wl_max'] = user_input(f"FWHM maximum wavelength (nm):", type_=str) +def fortuitous_variable_overlap(position, fortuitous_variables, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS): + """Return the closest full-field VSX variable matching a tracked pixel position.""" + try: + candidate = np.asarray(position, dtype=float).reshape(-1) + if candidate.size < 2 or not np.all(np.isfinite(candidate[:2])): + return None + duplicate_radius = max(float(duplicate_radius_pixels), 0.0) + except (TypeError, ValueError): + return None -def standard_filter(ld, observed_filter): - LimbDarkening.standard_list() + closest_match = None + for variable in fortuitous_variables or []: + try: + variable_position = np.asarray( + variable.get('pos', [variable.get('x'), variable.get('y')]), + dtype=float, + ).reshape(-1) + if variable_position.size < 2 or not np.all(np.isfinite(variable_position[:2])): + continue + except (AttributeError, TypeError, ValueError): + continue - while True: - if not observed_filter['filter']: - observed_filter['filter'] = user_input("\nPlease enter in the Filter Name or Abbreviation " - "(EX: Johnson V, V, STB, RJ): ", type_=str) + distance = float(np.hypot( + candidate[0] - variable_position[0], + candidate[1] - variable_position[1], + )) + if distance > duplicate_radius: + continue + if closest_match is None or distance < closest_match['distance_pixels']: + closest_match = { + 'variable': variable, + 'variable_name': str(variable.get('name') or 'unnamed VSX variable'), + 'variable_position': [float(variable_position[0]), float(variable_position[1])], + 'distance_pixels': distance, + } + return closest_match - if check_all_standard_filters(ld, observed_filter): - return - else: - log_info("\nError: The entered filter is not in the provided list of standard filters.", warn=True) - observed_filter['filter'] = None +def filter_comparison_stars_against_fortuitous_variables( + comparison_stars, + fortuitous_variables, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS): + """Remove every science comparison later identified by the full-field VSX search.""" + retained = [] + rejected = [] + for index, position in enumerate(comparison_stars or []): + try: + normalized_position = [float(position[0]), float(position[1])] + except (TypeError, ValueError, IndexError): + continue -def user_entered_ld(ld, observed_filter): - order = ['first', 'second', 'third', 'fourth'] + overlap = fortuitous_variable_overlap( + normalized_position, + fortuitous_variables, + duplicate_radius_pixels=duplicate_radius_pixels, + ) + if overlap is None: + retained.append(normalized_position) + continue - input_list = [(f"\nEnter in your {order[i]} nonlinear term:", - f"\nEnter in your {order[i]} nonlinear term uncertainty:") for i in range(len(order))] - ld_ = [(user_input(input_[0], type_=float), user_input(input_[1], type_=float)) for input_ in input_list] + rejected.append({ + 'comparison_index': index, + 'position': normalized_position, + **overlap, + }) + return retained, rejected - custom_range(ld, observed_filter) - ld.set_ld(ld_[0], ld_[1], ld_[2], ld_[3]) +def format_comp_star_coverage_text(summary): + coverage_text = ( + f"{summary['coverage_count']} valid frame(s)" + f" out of {summary.get('coverage_total_frame_count', 'n/a')} total" + f"; min_required={summary.get('coverage_min_required_count', 0)}" + ) + coverage_median = summary.get('coverage_reference_count', np.nan) + if np.isfinite(coverage_median): + coverage_text += f"; peer_median={coverage_median:.1f}" + return coverage_text + + +def format_comp_star_coverage_rejection_detail(summary): + coverage_count = int(summary.get('coverage_count', 0) or 0) + threshold_values = [] + for threshold in ( + summary.get('coverage_rejection_threshold_count'), + summary.get('coverage_min_required_count'), + ): + try: + numeric_threshold = float(threshold) + except (TypeError, ValueError): + continue + if np.isfinite(numeric_threshold): + threshold_values.append(int(np.ceil(numeric_threshold))) + + threshold_count = max(threshold_values) if threshold_values else None + if threshold_count is None: + detail_parts = [f"{coverage_count} valid frame(s)"] + elif coverage_count < threshold_count: + detail_parts = [f"{coverage_count} < {threshold_count} valid frame(s)"] + else: + detail_parts = [f"{coverage_count} valid frame(s); rejection threshold={threshold_count}"] -def nonlinear_ld(ld, info_dict): - user_entered = False - observed_filter = { - 'filter': info_dict['filter'], - 'name': None, - 'wl_min': info_dict['wl_min'], - 'wl_max': info_dict['wl_max'] + coverage_median = summary.get( + 'coverage_rejection_reference_count', + summary.get('coverage_reference_count', np.nan), + ) + try: + numeric_coverage_median = float(coverage_median) + except (TypeError, ValueError): + numeric_coverage_median = np.nan + if np.isfinite(numeric_coverage_median): + detail_parts.append(f"peer median={numeric_coverage_median:.1f}") + + return "; ".join(detail_parts) + + +def format_eebls_snr(value): + if value is None: + return "n/a" + + try: + numeric_value = float(value) + except (TypeError, ValueError): + return "n/a" + + return "n/a" if not np.isfinite(numeric_value) else f"{numeric_value:.2f}" + + +def extract_lightcurve_fit_transit_delta_bic(fit): + if fit is None: + return np.nan + + transit_qc = getattr(fit, 'transit_qc', None) + if isinstance(transit_qc, dict): + value = transit_qc.get('delta_bic', np.nan) + if np.isfinite(value): + return float(value) + + value = getattr(fit, 'transit_qc_delta_bic', np.nan) + return np.nan if not np.isfinite(value) else float(value) + + +def extract_lightcurve_fit_residual_scatter(fit): + if fit is None: + return np.nan + + transit_qc = getattr(fit, 'transit_qc', None) + if isinstance(transit_qc, dict): + value = transit_qc.get('residual_scatter', np.nan) + if np.isfinite(value): + return float(value) + + value = getattr(fit, 'transit_qc_residual_scatter', np.nan) + if np.isfinite(value): + return float(value) + + value = getattr(fit, 'res_stdev', np.nan) + if np.isfinite(value): + return float(value) + + residuals = np.asarray(getattr(fit, 'residuals', np.array([])), dtype=float) + data = np.asarray(getattr(fit, 'data', np.array([])), dtype=float) + if residuals.shape != data.shape or residuals.size == 0: + return np.nan + + median_flux = np.nanmedian(data) + if not np.isfinite(median_flux) or median_flux == 0: + return np.nan + return float(np.std(residuals) / median_flux) + + +def extract_lightcurve_fit_ktmf_metric(fit): + if fit is None: + return np.nan + + transit_qc = getattr(fit, 'transit_qc', None) + if isinstance(transit_qc, dict): + value = transit_qc.get('ktmf_metric', np.nan) + if np.isfinite(value): + return float(value) + + value = getattr(fit, 'transit_qc_ktmf_metric', np.nan) + return np.nan if not np.isfinite(value) else float(value) + + +def extract_lightcurve_fit_ktmf_contributions(fit): + if fit is None: + return [] + + transit_qc = getattr(fit, 'transit_qc', None) + if isinstance(transit_qc, dict): + contributions = transit_qc.get('ktmf_contributions') + if isinstance(contributions, list): + return contributions + + contributions = getattr(fit, 'transit_qc_ktmf_contributions', None) + return contributions if isinstance(contributions, list) else [] + + +def compact_comparison_attempt_for_output(attempt): + if not isinstance(attempt, dict): + return {} + + return { + 'rank': attempt.get('rank'), + 'comp_index': attempt.get('comp_index'), + 'label': attempt.get('label'), + 'selected': attempt.get('selected'), + 'selection_reason': attempt.get('selection_reason'), + 'selection_pass_ktmf_metric': attempt.get('selection_pass_ktmf_metric'), + 'selection_pass_transit_delta_bic': attempt.get('selection_pass_transit_delta_bic'), + 'selection_pass_eebls_snr': attempt.get('selection_pass_eebls_snr'), + 'selection_pass_residual_scatter': attempt.get('selection_pass_residual_scatter'), + 'target_model_scatter_basis': attempt.get('target_model_scatter_basis'), + 'projected_full_residual_scatter': attempt.get('projected_full_residual_scatter'), + 'selection_scatter': attempt.get('selection_scatter'), + 'selection_scatter_basis': attempt.get('selection_scatter_basis'), + 'target_comp_scatter': attempt.get('target_comp_scatter'), + 'selection_pass_target_comp_scatter': attempt.get('selection_pass_target_comp_scatter'), + 'selection_pass_transit_qc_status': attempt.get('selection_pass_transit_qc_status'), + 'selection_pass_transit_qc_summary': attempt.get('selection_pass_transit_qc_summary'), + 'scatter_gate_passed': attempt.get('scatter_gate_passed'), + 'scatter_gate_lowest_residual_scatter': attempt.get('scatter_gate_lowest_residual_scatter'), + 'scatter_gate_threshold': attempt.get('scatter_gate_threshold'), + 'scatter_adjusted_ktmf_metric': attempt.get('scatter_adjusted_ktmf_metric'), + 'combined_quality_ktmf_metric': attempt.get('combined_quality_ktmf_metric'), + 'combined_quality_best_residual_scatter': attempt.get('combined_quality_best_residual_scatter'), + 'combined_quality_best_target_comp_scatter': attempt.get('combined_quality_best_target_comp_scatter'), + 'combined_quality_best_comp_stability': attempt.get('combined_quality_best_comp_stability'), + 'final_refit_metric_note': attempt.get('final_refit_metric_note'), + 'ktmf_metric': attempt.get('ktmf_metric'), + 'ktmf_contributions': attempt.get('ktmf_contributions') or [], + 'transit_delta_bic': attempt.get('transit_delta_bic'), + 'eebls_snr': attempt.get('eebls_snr'), + 'residual_scatter': attempt.get('residual_scatter'), + 'fit_point_count': attempt.get('fit_point_count'), + 'transit_qc_status': attempt.get('transit_qc_status'), + 'transit_qc_summary': attempt.get('transit_qc_summary'), + 'rejected_by_transit_qc': attempt.get('rejected_by_transit_qc'), + 'failure_reason': attempt.get('failure_reason'), } - ld.check_fwhm(observed_filter) - if check_all_standard_filters(ld, observed_filter): - pass - elif observed_filter['wl_min'] and observed_filter['wl_max']: - custom_range(ld, observed_filter) - ld.set_filter('N/A', "Custom", float(observed_filter['wl_min']), float(observed_filter['wl_max'])) - else: - opt = user_input("\nWould you like EXOTIC to calculate your limb darkening parameters " - "with uncertainties? (y/n):", type_=str, values=['y', 'n']) - - if opt == 'y': - opt = user_input("Please enter 1 to use a standard filter or 2 for a customized filter:", - type_=int, values=[1, 2]) - if opt == 1: - observed_filter['filter'] = None - standard_filter(ld, observed_filter) - elif opt == 2: - custom_range(ld, observed_filter) - ld.set_filter('N/A', "Custom", float(observed_filter['wl_min']), float(observed_filter['wl_max'])) - else: - user_entered_ld(ld, observed_filter) - user_entered = True - if not user_entered: - ld.calculate_ld() +def record_comparison_attempt_selection_pass_metrics(attempt): + if not isinstance(attempt, dict): + return - info_dict['filter'] = ld.filter_name - info_dict['filter_desc'] = ld.filter_desc - info_dict['wl_min'] = ld.wl_min - info_dict['wl_max'] = ld.wl_max + attempt['selection_pass_ktmf_metric'] = attempt.get('ktmf_metric', np.nan) + attempt['selection_pass_transit_delta_bic'] = attempt.get('transit_delta_bic', np.nan) + attempt['selection_pass_eebls_snr'] = attempt.get('eebls_snr', np.nan) + attempt['selection_pass_residual_scatter'] = attempt.get('residual_scatter', np.nan) + attempt['selection_pass_selection_scatter'] = attempt.get('selection_scatter', np.nan) + attempt['selection_pass_target_comp_scatter'] = attempt.get('target_comp_scatter', np.nan) + attempt['selection_pass_fit_point_count'] = attempt.get('fit_point_count') + attempt['selection_pass_transit_qc_status'] = attempt.get('transit_qc_status') + attempt['selection_pass_transit_qc_summary'] = attempt.get('transit_qc_summary') -def get_ld_values(planet_dict, info_dict): - ld_obj = LimbDarkening(planet_dict) - nonlinear_ld(ld_obj, info_dict) +def comparison_selection_metric_changed(before, after, tolerance=5.0e-3): + try: + before = float(before) + after = float(after) + except (TypeError, ValueError): + return False + return np.isfinite(before) and np.isfinite(after) and abs(before - after) > tolerance + + +def update_selected_comparison_final_refit_note(selected_result): + if not isinstance(selected_result, dict): + return + + notes = [] + if comparison_selection_metric_changed( + selected_result.get('selection_pass_ktmf_metric'), + selected_result.get('ktmf_metric'), + ): + notes.append( + "KTMF " + f"{format_ktmf_metric(selected_result.get('selection_pass_ktmf_metric'))} -> " + f"{format_ktmf_metric(selected_result.get('ktmf_metric'))}" + ) + if comparison_selection_metric_changed( + selected_result.get('selection_pass_eebls_snr'), + selected_result.get('eebls_snr'), + tolerance=1.0e-2, + ): + selection_pass_eebls_snr = float(selected_result.get('selection_pass_eebls_snr')) + final_eebls_snr = float(selected_result.get('eebls_snr')) + notes.append( + "EEBLS SNR " + f"{selection_pass_eebls_snr:.2f} -> " + f"{final_eebls_snr:.2f}" + ) + if comparison_selection_metric_changed( + selected_result.get('selection_pass_transit_delta_bic'), + selected_result.get('transit_delta_bic'), + tolerance=1.0e-2, + ): + notes.append( + "Delta BIC " + f"{format_transit_delta_bic(selected_result.get('selection_pass_transit_delta_bic'))} -> " + f"{format_transit_delta_bic(selected_result.get('transit_delta_bic'))}" + ) - ld0 = ld_obj.ld0 - ld1 = ld_obj.ld1 - ld2 = ld_obj.ld2 - ld3 = ld_obj.ld3 - ld = [ld0[0], ld1[0], ld2[0], ld3[0]] + if not notes: + return - return ld, ld0, ld1, ld2, ld3 + selected_result['final_refit_metric_note'] = ( + "final full-resolution refit updated selected-candidate metrics: " + + "; ".join(notes) + ) + reason = selected_result.get('selection_reason') or '' + if selected_result['final_refit_metric_note'] not in reason: + selected_result['selection_reason'] = ( + f"{reason}; {selected_result['final_refit_metric_note']}" + if reason else selected_result['final_refit_metric_note'] + ) -def corruption_check(files): - valid_files = [] - for file in files: - plateStatus.setCurrentFilename(file) - try: - with fits.open(name=file, memmap=False, cache=False, lazy_load_hdus=False, ignore_missing_end=True) as hdu1: - valid_files.append(file) - except OSError as e: - # Since google collab can have problems with initial load of big data sets from google - # drive, lets pause and retry this once when we fail: if the file was corrupted the first time, - # nothing will get better... - log_info(f"Warning: retrying verify\n\t-File: {file}\n\t-Reason: {e}", warn=True) - sleep(5) - try: - with fits.open(name=file, memmap=False, cache=False, lazy_load_hdus=False, ignore_missing_end=True) as hdu1: - valid_files.append(file) - except OSError as e: - log.debug(f"Warning: corrupted file found and removed from reduction\n\t-File: {file}\n\t-Reason: {e}") - plateStatus.fitsFormatError(e) - return valid_files +def format_transit_delta_bic(value): + if value is None: + return "n/a" -def check_wcs(fits_file, save_directory, plate_opt, rt=False): - wcs_file = None + try: + numeric_value = float(value) + except (TypeError, ValueError): + return "n/a" - if plate_opt == 'y' and not rt: - wcs_file = get_wcs(fits_file, save_directory) - if not wcs_file: - if search_wcs(fits_file).is_celestial: - log_info("Your FITS files have WCS (World Coordinate System) information in their headers. " - "EXOTIC will proceed to use these. " - "NOTE: If you do not trust your WCS coordinates, " - "please restart EXOTIC after enabling plate solutions via astrometry.net.") - wcs_file = fits_file + return "n/a" if not np.isfinite(numeric_value) else f"{numeric_value:.2f}" + + +def format_residual_scatter(value): + if value is None: + return "n/a" + + try: + numeric_value = float(value) + except (TypeError, ValueError): + return "n/a" + + return "n/a" if not np.isfinite(numeric_value) else f"{numeric_value * 100.0:.4f}%" + + +def format_ktmf_metric(value): + if value is None: + return "n/a" + + try: + numeric_value = float(value) + except (TypeError, ValueError): + return "n/a" + + return "n/a" if not np.isfinite(numeric_value) else f"{numeric_value:.2f}/5.00" + + +def format_ktmf_contribution(contribution): + if not isinstance(contribution, dict): + return "KTMF contribution: unavailable" + + label = contribution.get('label', 'Unknown component') + detail = contribution.get('detail') or 'n/a' + available = bool(contribution.get('available')) + points = float(contribution.get('points', 0.0) or 0.0) + max_points = float(contribution.get('max_points', 0.0) or 0.0) + score = contribution.get('score', np.nan) + + if available and np.isfinite(score): + return ( + f"KTMF contribution: {label} +{points:.2f}/{max_points:.2f} " + f"(score={score:.2f}; {detail})" + ) + + return f"KTMF contribution: {label} +0.00/0.00 (unavailable; {detail})" + + +def summarize_lightcurve_fit_assessment(fit): + if fit is None: + return None + + fit_method = getattr(fit, 'ns_type', None) or getattr(fit, 'fit_method', None) or 'lm' + try: + rprs_retry_count = int(getattr(fit, 'rprs_posterior_refit_count', 0) or 0) + except (TypeError, ValueError): + rprs_retry_count = 0 + try: + ars_retry_count = int(getattr(fit, 'ars_posterior_refit_count', 0) or 0) + except (TypeError, ValueError): + ars_retry_count = 0 + try: + b_retry_count = int(getattr(fit, 'b_posterior_refit_count', 0) or 0) + except (TypeError, ValueError): + b_retry_count = 0 + + return { + 'fit_method': fit_method, + 'duration_prior_applied': bool(getattr(fit, 'duration_prior_applied', False)), + 'duration_prior_note': getattr(fit, 'duration_prior_note', None), + 'pre_ultranest_transit_coverage_valid': bool( + getattr(fit, 'pre_ultranest_transit_coverage_valid', False) + ), + 'pre_ultranest_transit_coverage_status': getattr( + fit, + 'pre_ultranest_transit_coverage_status', + None, + ), + 'pre_ultranest_transit_coverage_chance': getattr( + fit, + 'pre_ultranest_transit_coverage_chance', + np.nan, + ), + 'pre_ultranest_transit_coverage_note': getattr( + fit, + 'pre_ultranest_transit_coverage_note', + None, + ), + 'rprs_posterior_refit_applied': bool(getattr(fit, 'rprs_posterior_refit_applied', False)), + 'rprs_posterior_refit_count': rprs_retry_count, + 'rprs_posterior_refit_note': getattr(fit, 'rprs_posterior_refit_note', None), + 'ars_posterior_refit_applied': bool(getattr(fit, 'ars_posterior_refit_applied', False)), + 'ars_posterior_refit_count': ars_retry_count, + 'ars_posterior_refit_note': getattr(fit, 'ars_posterior_refit_note', None), + 'b_posterior_refit_applied': bool(getattr(fit, 'b_posterior_refit_applied', False)), + 'b_posterior_refit_count': b_retry_count, + 'b_posterior_refit_note': getattr(fit, 'b_posterior_refit_note', None), + 'sparse_posterior_live_point_extension_applied': bool( + getattr(fit, 'sparse_posterior_live_point_extension_applied', False) + ), + 'sparse_posterior_live_point_extension_note': getattr( + fit, + 'sparse_posterior_live_point_extension_note', + None, + ), + 'prefit_refinement_applied': bool(getattr(fit, 'prefit_refinement_applied', False)), + 'prefit_refinement_note': getattr(fit, 'prefit_refinement_note', None), + 'oot_baseline_parameter_fit_applied': bool( + getattr(fit, 'oot_baseline_parameter_fit_applied', False) + ), + 'oot_baseline_parameter_fit_note': getattr(fit, 'oot_baseline_parameter_fit_note', None), + 'oot_baseline_detrending_applied': bool(getattr(fit, 'oot_baseline_detrending_applied', False)), + 'oot_baseline_detrending_note': getattr(fit, 'oot_baseline_detrending_note', None), + 'airmass_fit_skipped': bool(getattr(fit, 'airmass_fit_skipped', False)), + 'airmass_correction_note': getattr(fit, 'airmass_correction_note', None), + 'nested_tmid_refinement_applied': bool(getattr(fit, 'nested_tmid_refinement_applied', False)), + 'nested_tmid_refinement_note': getattr(fit, 'nested_tmid_refinement_note', None), + 'ultranest_error_fallbacks': getattr(fit, 'ultranest_error_fallbacks', {}) or {}, + } - return wcs_file +def best_available_attempt_fit(attempt): + if not isinstance(attempt, dict): + return None + return ( + attempt.get('full_reduction_fit') + or attempt.get('fit') + or attempt.get('provisional_fit') + ) -def search_wcs(file): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=FITSFixedWarning) - header = fits.getheader(filename=file) - return WCS(header) - # return WCS(fits.open(file)[('SCI', 1)].header) +def log_lightcurve_fit_assessment_lines(fit, indent=" "): + assessment = summarize_lightcurve_fit_assessment(fit) + if not assessment: + return -def get_wcs(file, directory=""): - log_info("\nGetting the plate solution for your imaging file to translate pixel coordinates on the sky. " - "\nPlease wait....") - animate_toggle(True) - wcs_obj = PlateSolution(file=file, directory=directory) - wcs_file = wcs_obj.plate_solution() - animate_toggle() - return wcs_file + rprs_retry_status = "not applied" + if assessment['rprs_posterior_refit_applied']: + retry_count = assessment['rprs_posterior_refit_count'] + rprs_retry_status = ( + f"applied ({retry_count} refit(s))" + if retry_count > 0 else + "applied" + ) + ars_retry_status = "not applied" + if assessment['ars_posterior_refit_applied']: + retry_count = assessment['ars_posterior_refit_count'] + ars_retry_status = ( + f"applied ({retry_count} refit(s))" + if retry_count > 0 else + "applied" + ) + b_retry_status = "not applied" + if assessment['b_posterior_refit_applied']: + retry_count = assessment['b_posterior_refit_count'] + b_retry_status = ( + f"applied ({retry_count} refit(s))" + if retry_count > 0 else + "applied" + ) + prefit_status = "applied" if assessment['prefit_refinement_applied'] else "not applied" + oot_parameter_status = "applied" if assessment['oot_baseline_parameter_fit_applied'] else "not applied" + oot_status = "applied" if assessment['oot_baseline_detrending_applied'] else "not applied" + duration_prior_status = "applied" if assessment['duration_prior_applied'] else "not applied" + sparse_extension_status = ( + "applied" + if assessment['sparse_posterior_live_point_extension_applied'] + else "not applied" + ) + log_info( + f"{indent}fit assessment: fit_method={assessment['fit_method']}, " + f"duration_prior={duration_prior_status}, " + f"Rp/R* posterior retry={rprs_retry_status}, " + f"a/Rs posterior retry={ars_retry_status}, " + f"impact parameter posterior retry={b_retry_status}, " + f"sparse posterior extension={sparse_extension_status}, " + f"prefit_refinement={prefit_status}, " + f"oot_baseline_parameter_fit={oot_parameter_status}, " + f"oot_baseline_detrending={oot_status}" + ) + if assessment.get('duration_prior_note'): + log_info(f"{indent}Duration prior note: {assessment['duration_prior_note']}") + if assessment.get('pre_ultranest_transit_coverage_note'): + status = assessment.get('pre_ultranest_transit_coverage_status') or 'unknown' + chance = coerce_finite_transit_qc_scalar( + assessment.get('pre_ultranest_transit_coverage_chance', np.nan) + ) + chance_text = f", chance~{100.0 * float(chance):.0f}%" if np.isfinite(chance) else "" + log_info( + f"{indent}Pre-UltraNest coverage note: status={str(status).upper()}{chance_text}; " + f"{assessment['pre_ultranest_transit_coverage_note']}" + ) + if assessment.get('rprs_posterior_refit_note'): + log_info(f"{indent}Rp/R* posterior retry note: {assessment['rprs_posterior_refit_note']}") + if assessment.get('ars_posterior_refit_note'): + log_info(f"{indent}a/Rs posterior retry note: {assessment['ars_posterior_refit_note']}") + if assessment.get('b_posterior_refit_note'): + log_info(f"{indent}Impact parameter posterior retry note: {assessment['b_posterior_refit_note']}") + if assessment.get('sparse_posterior_live_point_extension_note'): + log_info( + f"{indent}Sparse posterior live-point extension note: " + f"{assessment['sparse_posterior_live_point_extension_note']}" + ) + if assessment.get('prefit_refinement_note'): + log_info(f"{indent}Prefit refinement note: {assessment['prefit_refinement_note']}") + if assessment.get('oot_baseline_parameter_fit_note'): + log_info(f"{indent}OOT baseline parameter-fit note: {assessment['oot_baseline_parameter_fit_note']}") + if assessment.get('oot_baseline_detrending_note'): + log_info(f"{indent}OOT baseline detrending note: {assessment['oot_baseline_detrending_note']}") + if assessment.get('airmass_correction_note'): + log_info(f"{indent}Airmass correction note: {assessment['airmass_correction_note']}") + if assessment.get('nested_tmid_refinement_note'): + log_info(f"{indent}Nested Tmid refinement note: {assessment['nested_tmid_refinement_note']}") + if assessment.get('ultranest_error_fallbacks'): + fallback_keys = ", ".join(sorted(assessment['ultranest_error_fallbacks'])) + log_info( + f"{indent}UltraNest uncertainty fallback note: replaced posterior summary " + f"error(s) for {fallback_keys} using the sampled log-likelihood neighborhood." + ) -# Getting the right ascension and declination for every pixel in imaging file if there is a plate solution -def get_ra_dec(header): - wcs_header = WCS(header) - xaxis = np.arange(header['NAXIS1']) - yaxis = np.arange(header['NAXIS2']) - x, y = np.meshgrid(xaxis, yaxis) - return wcs_header.all_pix2world(x, y, 1) +def log_comparison_candidate_evaluation_start(comp_summary, rank, ranked_count, method_label, fit_diagnostics): + if comp_summary is None: + return + + label = comp_summary.get('label') + if label is None: + comp_index = comp_summary.get('comp_index') + label = "comparison candidate" if comp_index is None else f"Comp {comp_index + 1}" + position_text = format_comp_star_position(comp_summary.get('position')) + coverage_text = format_comp_star_coverage_text({ + 'coverage_count': comp_summary.get('coverage_count', 0), + 'coverage_total_frame_count': comp_summary.get('coverage_total_frame_count', 0), + 'coverage_reference_count': comp_summary.get('coverage_reference_count', np.nan), + 'coverage_min_required_count': comp_summary.get('coverage_min_required_count', 0), + }) + suitability_score = comp_summary.get('aggregate_score', np.nan) + suitability_text = "n/a" if not np.isfinite(suitability_score) else f"{suitability_score * 100.0:.4f}%" + usable_point_count = 0 if fit_diagnostics is None else fit_diagnostics.get('usable_point_count', 0) + ensemble_frame_rejected_count = int(comp_summary.get('ensemble_frame_rejected_count', 0) or 0) + + log_info( + f"\nStarting comparison-star target-fit evaluation for {label} ({position_text}) " + f"[rank {rank + 1}/{ranked_count}] with {method_label}." + ) + log_info( + f" Candidate inputs: suitability={suitability_text}, coverage={coverage_text}, " + f"usable_after_filters={usable_point_count}." + ) + if ensemble_frame_rejected_count > 0: + log_info( + " Candidate intercomparison clipping rejects " + f"{ensemble_frame_rejected_count} comparison-unstable frame(s) before target fitting." + ) + log_info(" Preparing comparison-candidate light curve for the full reduction.") + + +def log_comparison_candidate_evaluation_result(attempt): + if not attempt: + return + + fit = best_available_attempt_fit(attempt) + fit_method = "n/a" + assessment = summarize_lightcurve_fit_assessment(fit) + if assessment: + fit_method = assessment.get('fit_method', 'n/a') + + qc_status = attempt.get('transit_qc_status') + qc_text = "n/a" if not qc_status else str(qc_status).upper() + if attempt.get('fit') is None: + status_text = "FAILED" + elif attempt.get('rejected_by_transit_qc', False): + status_text = f"REJECTED ({qc_text})" + elif attempt.get('full_reduction_applied', False): + status_text = f"COMPLETE ({qc_text})" + else: + status_text = "PROVISIONAL ONLY" -def deg_to_pix(exp_ra, exp_dec, ra_list, dec_list): - dist = (ra_list - exp_ra) ** 2 + (dec_list - exp_dec) ** 2 - return np.unravel_index(dist.argmin(), dist.shape) + reason_text = ( + attempt.get('transit_qc_summary') + or attempt.get('failure_reason') + or attempt.get('selection_reason') + or "completed comparison-star target-fit evaluation." + ) + log_info( + f"Completed comparison-star target-fit evaluation for {attempt.get('label', 'comparison candidate')}: " + f"status={status_text}, fit_method={fit_method}, fit_points={attempt.get('fit_point_count', 0)}, " + f"transit_qc={qc_text}, transit_delta_bic={format_transit_delta_bic(attempt.get('transit_delta_bic', np.nan))}, " + f"residual_scatter={format_residual_scatter(attempt.get('residual_scatter', np.nan))}, " + f"ktmf={format_ktmf_metric(attempt.get('ktmf_metric', np.nan))}, reason={reason_text}" + ) + parameter_summary = attempt.get('parameter_summary') + if parameter_summary: + log_info(f" parameters: {parameter_summary}") + log_lightcurve_fit_assessment_lines(fit, indent=" ") + if attempt.get('final_output_dir'): + log_info(f" outputs: {attempt['final_output_dir']}") + for contribution in attempt.get('ktmf_contributions', []): + log_info(f" {format_ktmf_contribution(contribution)}") + + +def comparison_selection_metric_label(selection_metric): + if selection_metric == 'first_qc_pass': + return "First QC PASS" + if selection_metric == 'promising_partial': + return "Promising Partial" + if selection_metric == 'stellar_variability_scatter': + return "Out-of-transit scatter" + if selection_metric == 'stellar_variability_ensemble': + return "Calibrated stellar-variability ensemble" + if selection_metric == 'comparison_field_rank': + return "Comparison-Field Rank" + if selection_metric == 'ktmf': + return "KTMF" + if selection_metric == 'ktmf_scatter': + return "KTMF / scatter" + if selection_metric == 'ktmf_combined_quality': + return "KTMF / projected scatter" + if selection_metric == 'eebls_snr': + return "EEBLS SNR" + return "transit-vs-flat Delta BIC" + + +def should_stop_after_promising_partial_comparison_attempt(attempt): + if not isinstance(attempt, dict): + return False + if attempt.get('fit') is None or not attempt.get('full_reduction_applied', False): + return False + if attempt.get('rejected_by_transit_qc', False): + return False + status = str(attempt.get('transit_qc_status') or '').strip().lower() + if status != 'marginal': + return False -def check_target_pixel_wcs(input_x_pixel, input_y_pixel, info_dict, ra_list, dec_list, image_data, obs_time): - """ - Verify the provided pixel coordinates match the target's right ascension and declination. - """ - updated_ra, updated_dec = update_coordinates_with_proper_motion(info_dict, obs_time) + try: + coverage_priority = int(attempt.get('preflight_coverage_priority', 4)) + except (TypeError, ValueError): + coverage_priority = 4 + if coverage_priority > 2: + return False - calculated_y_pixel, calculated_x_pixel = deg_to_pix(updated_ra, updated_dec, ra_list, dec_list) + ktmf_metric = coerce_finite_transit_qc_scalar(attempt.get('ktmf_metric', np.nan)) + delta_bic = coerce_finite_transit_qc_scalar(attempt.get('transit_delta_bic', np.nan)) + return ( + np.isfinite(ktmf_metric) + and ktmf_metric >= PROMISING_PARTIAL_COMPARISON_KTMF_MIN + and np.isfinite(delta_bic) + and delta_bic >= TRANSIT_QC_DELTA_BIC_PASS_THRESHOLD + ) - centroid_x, centroid_y, sigma_x, sigma_y = get_psf_parameters(image_data, calculated_x_pixel, calculated_y_pixel) - return check_coordinates(input_x_pixel, input_y_pixel, centroid_x, centroid_y, sigma_x, sigma_y, - calculated_x_pixel, calculated_y_pixel) +def scatter_gate_comparison_attempts( + attempts, + max_scatter_multiplier=COMPARISON_SELECTION_MAX_SCATTER_MULTIPLIER): + attempts = list(attempts or []) + finite_scatters = [ + float(attempt.get('selection_scatter', attempt.get('residual_scatter', np.nan))) + for attempt in attempts + if np.isfinite(attempt.get('selection_scatter', attempt.get('residual_scatter', np.nan))) + ] + if not finite_scatters: + for attempt in attempts: + attempt['scatter_gate_passed'] = True + attempt['scatter_gate_lowest_residual_scatter'] = np.nan + attempt['scatter_gate_threshold'] = np.nan + return attempts, np.nan, np.nan + + lowest_scatter = min(finite_scatters) + scatter_threshold = lowest_scatter * float(max_scatter_multiplier) + eligible_attempts = [] + for attempt in attempts: + residual_scatter = attempt.get('selection_scatter', attempt.get('residual_scatter', np.nan)) + scatter_passed = ( + np.isfinite(residual_scatter) + and residual_scatter <= scatter_threshold + ) + attempt['scatter_gate_passed'] = bool(scatter_passed) + attempt['scatter_gate_lowest_residual_scatter'] = lowest_scatter + attempt['scatter_gate_threshold'] = scatter_threshold + if scatter_passed: + eligible_attempts.append(attempt) + + return eligible_attempts or attempts, lowest_scatter, scatter_threshold + + +def finite_positive_attempt_values(attempts, key): + values = [] + for attempt in attempts: + value = attempt.get(key, np.nan) + if np.isfinite(value) and value > 0: + values.append(float(value)) + return values + + +def comparison_attempt_combined_quality_ktmf(attempt): + ktmf_metric = attempt.get('ktmf_metric', np.nan) + if not np.isfinite(ktmf_metric): + return np.nan + + selection_scatter = attempt.get('selection_scatter', np.nan) + if ( + np.isfinite(selection_scatter) + and selection_scatter > 0 + ): + return float(ktmf_metric) / float(selection_scatter * 100.0) + return float(ktmf_metric) + + +def annotate_comparison_attempt_combined_quality_scores(attempts): + attempts = list(attempts or []) + for attempt in attempts: + selection_scatter = attempt.get('selection_scatter', np.nan) + if not np.isfinite(selection_scatter): + residual_scatter = attempt.get('residual_scatter', np.nan) + if np.isfinite(residual_scatter): + attempt['selection_scatter'] = residual_scatter + attempt.setdefault( + 'selection_scatter_basis', + "candidate UltraNest model residual scatter", + ) + + best_residual_scatter = min(finite_positive_attempt_values(attempts, 'selection_scatter'), default=np.nan) + for attempt in attempts: + attempt['combined_quality_best_residual_scatter'] = best_residual_scatter + attempt['combined_quality_best_target_comp_scatter'] = np.nan + attempt['combined_quality_best_comp_stability'] = np.nan + attempt['combined_quality_ktmf_metric'] = comparison_attempt_combined_quality_ktmf(attempt) + attempt['scatter_adjusted_ktmf_metric'] = attempt['combined_quality_ktmf_metric'] + + return attempts + + +def comparison_attempt_ranking_score(attempt): + value = attempt.get('combined_quality_ktmf_metric', np.nan) + if np.isfinite(value): + return value + value = attempt.get('scatter_adjusted_ktmf_metric', np.nan) + if np.isfinite(value): + return value + value = attempt.get('ktmf_metric', np.nan) + if np.isfinite(value): + return value + return np.nan + + +def select_preferred_comparison_attempt(attempts, pick_comparison_by_eebls_snr=True): + selected_result = None + selection_metric = 'ktmf' + if not attempts: + return selected_result, selection_metric + + attempts, _, _ = scatter_gate_comparison_attempts(attempts) + annotate_comparison_attempt_combined_quality_scores(attempts) + has_ktmf = any(np.isfinite(attempt.get('ktmf_metric', np.nan)) for attempt in attempts) + has_eebls = any(np.isfinite(attempt.get('eebls_snr', np.nan)) for attempt in attempts) + has_delta_bic = any(np.isfinite(attempt.get('transit_delta_bic', np.nan)) for attempt in attempts) + + if has_ktmf: + selection_metric = 'ktmf_combined_quality' + if pick_comparison_by_eebls_snr: + selected_result = min( + attempts, + key=lambda attempt: ( + 0 if np.isfinite(comparison_attempt_ranking_score(attempt)) else 1, + -comparison_attempt_ranking_score(attempt) + if np.isfinite(comparison_attempt_ranking_score(attempt)) else np.inf, + 0 if np.isfinite(attempt.get('ktmf_metric', np.nan)) else 1, + -attempt.get('ktmf_metric', np.nan) if np.isfinite(attempt.get('ktmf_metric', np.nan)) else np.inf, + 0 if np.isfinite(attempt.get('eebls_snr', np.nan)) else 1, + -attempt.get('eebls_snr', np.nan) if np.isfinite(attempt.get('eebls_snr', np.nan)) else np.inf, + 0 if np.isfinite(attempt.get('transit_delta_bic', np.nan)) else 1, + -attempt.get('transit_delta_bic', np.nan) if np.isfinite(attempt.get('transit_delta_bic', np.nan)) else np.inf, + attempt.get('rank', np.inf), + ), + ) + else: + selected_result = min( + attempts, + key=lambda attempt: ( + 0 if np.isfinite(comparison_attempt_ranking_score(attempt)) else 1, + -comparison_attempt_ranking_score(attempt) + if np.isfinite(comparison_attempt_ranking_score(attempt)) else np.inf, + 0 if np.isfinite(attempt.get('ktmf_metric', np.nan)) else 1, + -attempt.get('ktmf_metric', np.nan) if np.isfinite(attempt.get('ktmf_metric', np.nan)) else np.inf, + 0 if np.isfinite(attempt.get('transit_delta_bic', np.nan)) else 1, + -attempt.get('transit_delta_bic', np.nan) if np.isfinite(attempt.get('transit_delta_bic', np.nan)) else np.inf, + 0 if np.isfinite(attempt.get('eebls_snr', np.nan)) else 1, + -attempt.get('eebls_snr', np.nan) if np.isfinite(attempt.get('eebls_snr', np.nan)) else np.inf, + attempt.get('rank', np.inf), + ), + ) + elif pick_comparison_by_eebls_snr and has_eebls: + selection_metric = 'eebls_snr' + selected_result = min( + attempts, + key=lambda attempt: ( + 0 if np.isfinite(attempt.get('eebls_snr', np.nan)) else 1, + -attempt.get('eebls_snr', np.nan) if np.isfinite(attempt.get('eebls_snr', np.nan)) else np.inf, + 0 if np.isfinite(attempt.get('transit_delta_bic', np.nan)) else 1, + -attempt.get('transit_delta_bic', np.nan) if np.isfinite(attempt.get('transit_delta_bic', np.nan)) else np.inf, + attempt.get('rank', np.inf), + ), + ) + elif has_delta_bic: + selection_metric = 'transit_delta_bic' + selected_result = min( + attempts, + key=lambda attempt: ( + 0 if np.isfinite(attempt.get('transit_delta_bic', np.nan)) else 1, + -attempt.get('transit_delta_bic', np.nan) if np.isfinite(attempt.get('transit_delta_bic', np.nan)) else np.inf, + 0 if np.isfinite(attempt.get('eebls_snr', np.nan)) else 1, + -attempt.get('eebls_snr', np.nan) if np.isfinite(attempt.get('eebls_snr', np.nan)) else np.inf, + attempt.get('rank', np.inf), + ), + ) + elif has_eebls: + selection_metric = 'eebls_snr' + selected_result = min( + attempts, + key=lambda attempt: ( + 0 if np.isfinite(attempt.get('eebls_snr', np.nan)) else 1, + -attempt.get('eebls_snr', np.nan) if np.isfinite(attempt.get('eebls_snr', np.nan)) else np.inf, + attempt.get('rank', np.inf), + ), + ) + else: + selected_result = min(attempts, key=lambda attempt: attempt.get('rank', np.inf)) + return selected_result, selection_metric -def get_psf_parameters(image_data, x_pixel, y_pixel): - psf_data = fit_centroid(image_data, [x_pixel, y_pixel], 0) - return psf_data[0], psf_data[1], psf_data[3], psf_data[4] +def target_fit_candidate_identity(candidate): + return ( + candidate.get('method'), + candidate.get('a'), + candidate.get('an'), + candidate.get('comp_index'), + ) -def check_coordinates(input_x_pixel, input_y_pixel, centroid_x, centroid_y, sigma_x, sigma_y, - calculated_x_pixel, calculated_y_pixel): - while True: - try: - validate_pixel_coordinates(input_x_pixel, input_y_pixel, centroid_x, centroid_y, sigma_x, sigma_y) - return input_x_pixel, input_y_pixel - except ValueError: - new_x_pixel, new_y_pixel = prompt_user_for_coordinates(input_x_pixel, input_y_pixel, - calculated_x_pixel, calculated_y_pixel) - if new_x_pixel == input_x_pixel and new_y_pixel == input_y_pixel: - return input_x_pixel, input_y_pixel - else: - input_x_pixel, input_y_pixel = new_x_pixel, new_y_pixel +def summarize_target_fit_candidate(candidate, fit_meta, comp_stars): + comp_index = candidate.get('comp_index') + fit_meta = {} if fit_meta is None else dict(fit_meta) + fit_result = fit_meta.get('myfit') + return { + 'label': "Target-only" if comp_index is None else f"Comp {comp_index + 1}", + 'position': None if comp_index is None else comp_stars[comp_index], + 'selected': False, + 'method_label': comparison_method_label(candidate), + 'prescore': candidate.get('prescore', np.inf), + 'fit': fit_result, + 'coverage_count': candidate.get('coverage_count', 0), + 'coverage_total_frame_count': candidate.get('coverage_total_frame_count', 0), + 'coverage_reference_count': candidate.get('coverage_reference_count', np.nan), + 'coverage_min_required_count': candidate.get('coverage_min_required_count', 0), + 'coverage_rejected': candidate.get('coverage_rejected', False), + 'fit_point_count': fit_meta.get('fit_point_count', 0), + 'eebls_snr': fit_meta.get('eebls_snr', np.nan), + 'transit_delta_bic': fit_meta.get('transit_delta_bic', np.nan), + 'residual_scatter': fit_meta.get('residual_scatter', np.nan), + 'ktmf_metric': fit_meta.get('ktmf_metric', np.nan), + 'ktmf_contributions': fit_meta.get('ktmf_contributions') or [], + 'fit_diagnostics': fit_meta.get('fit_diagnostics') or {}, + 'failure_reason': fit_meta.get('failure_reason'), + 'parameter_summary': summarize_lightcurve_fit_parameters(fit_result), + 'comp_index': comp_index, + 'a': candidate.get('a'), + 'an': candidate.get('an'), + 'method': candidate.get('method'), + 'candidate_order': candidate.get('candidate_order'), + 'transit_qc_status': fit_meta.get('transit_qc_status'), + 'transit_qc_summary': fit_meta.get('transit_qc_summary'), + 'rejected_by_transit_qc': fit_meta.get('rejected_by_transit_qc', False), + } -def validate_pixel_coordinates(input_x_pixel, input_y_pixel, centroid_x, centroid_y, sigma_x, sigma_y): - """ - Validating the provided pixel coordinates are within 5 PSF of the expected coordinates. - """ - x_min = centroid_x - (sigma_x * 5) - x_max = centroid_x + (sigma_x * 5) - y_min = centroid_y - (sigma_y * 5) - y_max = centroid_y + (sigma_y * 5) - if not (x_min <= input_x_pixel <= x_max): - log_info("\nWarning: The X Pixel Coordinate entered does not match the target's Right Ascension.", warn=True) - raise ValueError - if not (y_min <= input_y_pixel <= y_max): - log_info("\nWarning: The Y Pixel Coordinate entered does not match the target's Declination.", warn=True) - raise ValueError +def log_comparison_calibration_fit_attempt_summaries(attempts, method_label): + if not attempts: + return + + log_info("\nComparison-star calibration target-fit diagnostics:") + log_info(f"Photometry method: {method_label}") + + for attempt in attempts: + selected_label = " [selected]" if attempt.get('selected') else "" + position_text = format_comp_star_position(attempt.get('position')) + diagnostics = attempt.get('fit_diagnostics') or {} + usable_point_count = diagnostics.get('usable_point_count', 0) + coverage_text = format_comp_star_coverage_text(attempt) + suitability_score = attempt.get('aggregate_score', np.inf) + suitability_text = "n/a" if not np.isfinite(suitability_score) else f"{suitability_score * 100.0:.4f}%" + eebls_text = format_eebls_snr(attempt.get('eebls_snr', np.nan)) + transit_delta_bic_text = format_transit_delta_bic(attempt.get('transit_delta_bic', np.nan)) + residual_text = format_residual_scatter(attempt.get('residual_scatter', np.nan)) + ktmf_text = format_ktmf_metric(attempt.get('ktmf_metric', np.nan)) + reason_text = attempt.get('selection_reason') or attempt.get( + 'failure_reason', + "selected: strongest KTMF among the evaluated comparison stars", + ) + if attempt.get('failed_run_dir'): + reason_text += f"; archived={attempt['failed_run_dir']}" + log_info( + f" {attempt['label']}{selected_label} ({position_text}): " + f"suitability={suitability_text}, coverage={coverage_text}, " + f"usable_after_filters={usable_point_count}, fit_points={attempt.get('fit_point_count', 0)}, " + f"eebls_snr={eebls_text}, transit_delta_bic={transit_delta_bic_text}, " + f"residual_scatter={residual_text}, ktmf={ktmf_text}, reason={reason_text}" + ) + parameter_summary = attempt.get('parameter_summary') + if parameter_summary: + log_info(f" parameters: {parameter_summary}") + log_lightcurve_fit_assessment_lines(best_available_attempt_fit(attempt), indent=" ") + for contribution in attempt.get('ktmf_contributions', []): + log_info(f" {format_ktmf_contribution(contribution)}") + + +def log_target_fit_candidate_summaries(candidate_summaries, max_entries=10): + if not candidate_summaries: + return + + log_info("\nTarget-fit candidate diagnostics:") + + displayed_summaries = candidate_summaries[:max_entries] + for summary in displayed_summaries: + selected_label = " [selected]" if summary.get('selected') else "" + position_text = format_comp_star_position(summary.get('position')) + diagnostics = summary.get('fit_diagnostics') or {} + usable_point_count = diagnostics.get('usable_point_count', 0) + coverage_text = format_comp_star_coverage_text(summary) + prescore = summary.get('prescore', np.inf) + prescore_text = "n/a" if not np.isfinite(prescore) else f"{prescore * 100.0:.4f}%" + eebls_text = format_eebls_snr(summary.get('eebls_snr', np.nan)) + transit_delta_bic_text = format_transit_delta_bic(summary.get('transit_delta_bic', np.nan)) + residual_text = format_residual_scatter(summary.get('residual_scatter', np.nan)) + ktmf_text = format_ktmf_metric(summary.get('ktmf_metric', np.nan)) + reason_text = summary.get( + 'failure_reason', + "selected: strongest KTMF in the evaluated candidate set", + ) + log_info( + f" {summary['label']}{selected_label} ({position_text}) with {summary['method_label']}: " + f"prescore={prescore_text}, coverage={coverage_text}, " + f"usable_after_filters={usable_point_count}, fit_points={summary.get('fit_point_count', 0)}, " + f"eebls_snr={eebls_text}, transit_delta_bic={transit_delta_bic_text}, " + f"residual_scatter={residual_text}, ktmf={ktmf_text}, reason={reason_text}" + ) + parameter_summary = summary.get('parameter_summary') + if parameter_summary: + log_info(f" parameters: {parameter_summary}") + for contribution in summary.get('ktmf_contributions', []): + log_info(f" {format_ktmf_contribution(contribution)}") + + if len(candidate_summaries) > len(displayed_summaries): + log_info( + f" ... omitted {len(candidate_summaries) - len(displayed_summaries)} additional " + "target-fit candidate(s); consider increasing the log limit if you need the full list." + ) -def prompt_user_for_coordinates(input_x_pixel, input_y_pixel, calculated_x_pixel, calculated_y_pixel): - log_info(f"Your input pixel coordinates: [{input_x_pixel}, {input_y_pixel}]") - log_info(f"EXOTIC's calculated pixel coordinates: [{calculated_x_pixel}, {calculated_y_pixel}]") - opt = user_input("Would you like to re-enter the pixel coordinates? (y/n): ", type_=str, values=['y', 'n']) +def comparison_calibration_selection_reason(summary, best_comp_score): + if summary.get('selected'): + return "selected: lowest suitability score among coverage-qualified, sigma-clip-qualified comparison stars for this method" - if opt == 'y': - use_suggested = user_input( - f"Here are the suggested pixel coordinates:" - f" X Pixel: {calculated_x_pixel}" - f" Y Pixel: {calculated_y_pixel}" - "\nWould you like to use these? (y/n): ", - type_=str, values=['y', 'n'] + if summary.get('coverage_rejected'): + return ( + "not selected: low coverage " + f"({format_comp_star_coverage_rejection_detail(summary)})" ) - if use_suggested == 'y': - return calculated_x_pixel, calculated_y_pixel - else: - input_x_pixel = user_input("Please re-enter the target star's X Pixel Coordinate: ", type_=int) - input_y_pixel = user_input("Please re-enter the target star's Y Pixel Coordinate: ", type_=int) - - return input_x_pixel, input_y_pixel + if summary.get('suitability_outlier_rejected'): + threshold = summary.get('suitability_high_threshold', np.nan) + if np.isfinite(threshold): + return ( + "not selected: suitability score was rejected by high-side sigma clipping " + f"({summary['aggregate_score'] * 100.0:.4f}% > {threshold * 100.0:.4f}%)" + ) + return "not selected: suitability score was rejected by high-side sigma clipping" + + aggregate_score = summary.get('aggregate_score', np.inf) + if not np.isfinite(aggregate_score): + return "not selected: no usable ensemble or pairwise calibration score" + + if np.isfinite(best_comp_score): + score_gap = aggregate_score - best_comp_score + if np.isfinite(score_gap) and score_gap > 0: + return ( + "not selected: suitability score was " + f"{score_gap * 100.0:.4f}% above the selected comparison star" + ) + return "not selected: another comparison star ranked better for this photometry method" + + +def comparison_candidate_fit_selection_reason(summary, photometry_info): + if summary.get('failure_reason'): + return summary['failure_reason'] + + selection_basis = photometry_info.get('selection_basis', 'target_fit') + selection_metric = photometry_info.get('selection_metric', 'ktmf') + selected_comp_num = photometry_info.get('comp_star_num') + selected_eebls_snr = photometry_info.get('comparison_eebls_snr', np.nan) + selected_transit_delta_bic = photometry_info.get('comparison_transit_delta_bic', np.nan) + selected_ktmf_metric = photometry_info.get('comparison_ktmf_metric', np.nan) + candidate_eebls_snr = summary.get('eebls_snr', np.nan) + candidate_transit_delta_bic = summary.get('transit_delta_bic', np.nan) + candidate_ktmf_metric = summary.get('ktmf_metric', np.nan) + + if summary.get('selected'): + if selection_basis == 'comparison_field': + return "selected: comparison-field calibration ranked this star best for the chosen photometry method" + if selection_basis == 'comparison_field_retry': + return ( + "selected: comparison-field calibration fell back to this star " + "after better-ranked candidates failed target fitting" + ) + if selection_basis == 'comparison_field_qc_fallback': + return ( + "selected: best available comparison-star fit after all completed candidates were rejected " + "by transit QC" + ) + if selection_metric == 'first_qc_pass': + return "selected: first completed comparison-star candidate with PASS transit QC" + if selection_metric == 'ktmf' and np.isfinite(candidate_ktmf_metric): + return "selected: highest KTMF in the chosen search" + if selection_metric == 'eebls_snr' and np.isfinite(candidate_eebls_snr): + return "selected: highest EEBLS SNR in the chosen search" + if np.isfinite(candidate_transit_delta_bic): + return "selected: strongest transit-vs-flat Delta BIC in the chosen search" + return "selected: strongest transit evidence in the chosen search" + + if selection_basis == 'comparison_field': + if selected_comp_num is None: + return "not selected: comparison-field calibration chose a different candidate" + return f"not selected: comparison-field calibration chose Comp {selected_comp_num}" + if selection_basis == 'comparison_field_retry': + if selected_comp_num is None: + return "not selected: comparison-field fallback chose a different candidate" + return ( + "not selected: comparison-field fallback chose " + f"Comp {selected_comp_num} after better-ranked candidate(s) failed target fitting" + ) + if selection_basis == 'comparison_field_qc_fallback': + if selected_comp_num is None: + return "not selected: comparison-field QC fallback chose a different candidate" + return ( + "not selected: comparison-field QC fallback chose " + f"Comp {selected_comp_num} as the best available rejected fit" + ) + if selection_metric == 'first_qc_pass': + if selected_comp_num is None: + return "not selected: search stopped after another candidate reached PASS transit QC" + return ( + "not selected: search stopped after " + f"Comp {selected_comp_num} reached PASS transit QC" + ) -# Checks if comparison star is variable via querying SIMBAD -def query_variable_star_apis(ra, dec): - # Convert comparison star coordinates from pixel to WCS - sample = SkyCoord(ra * u.deg, dec * u.deg, frame='fk5') - return vsx_variable(sample.ra.deg, sample.dec.deg) - # radius = u.Quantity(20.0, u.arcsec) - # # Query GAIA first to check for variability using the phot_variable_flag trait - # gaia_result = gaia_query(sample, radius) - # if not gaia_result: - # log_info("Warning: Your comparison star cannot be resolved in the Gaia star database; " - # "EXOTIC cannot check if it is variable or not. " - # "\nEXOTIC will still include this star in the reduction. " - # "\nPlease proceed with caution as we cannot check for stellar variability.\n", warn=True) - # else: - # # Individually go through the phot_variable_flag indicator for each star to see if variable or not - # variableFlagList = gaia_result.columns["phot_variable_flag"] - # constantCounter = 0 - # for currFlag in variableFlagList: - # if currFlag == "VARIABLE": - # return True - # elif currFlag == "NOT_AVAILABLE": - # continue - # elif currFlag == "CONSTANT": - # constantCounter += 1 - # if constantCounter == len(variableFlagList): - # return False - # - # # Query SIMBAD and search identifier result table to determine if comparison star is variable in any form - # # This is a secondary check if GAIA query returns inconclusive results - # star_name = simbad_query(sample) - # if not star_name: - # log_info("Warning: Your comparison star cannot be resolved in the SIMBAD star database; " - # "EXOTIC cannot check if it is variable or not. " - # "\nEXOTIC will still include this star in the reduction. " - # "\nPlease proceed with caution as we cannot check for stellar variability.\n", warn=True) - # return False - # else: - # identifiers = Simbad.query_objectids(star_name) - # - # for currName in identifiers: - # if "V*" in currName[0]: - # return True - # return False + if selection_metric == 'eebls_snr' and np.isfinite(selected_eebls_snr): + if not np.isfinite(candidate_eebls_snr): + return "not selected: no finite EEBLS SNR was available for this candidate" + if candidate_eebls_snr < selected_eebls_snr - 1e-12: + return ( + "not selected: EEBLS SNR was " + f"{candidate_eebls_snr:.2f} vs {selected_eebls_snr:.2f} for the selected fit" + ) + if candidate_eebls_snr > selected_eebls_snr + 1e-12: + return ( + "not selected: this post-selection diagnostic fit has a stronger " + "EEBLS box signal than the selected fit; the earlier search did not choose it" + ) + if np.isfinite(candidate_ktmf_metric) and np.isfinite(selected_ktmf_metric): + if candidate_ktmf_metric < selected_ktmf_metric - 1e-12: + return ( + "not selected: KTMF was " + f"{candidate_ktmf_metric:.2f} vs {selected_ktmf_metric:.2f} for the selected fit" + ) + if candidate_ktmf_metric > selected_ktmf_metric + 1e-12: + return ( + "not selected: this post-selection diagnostic fit has a higher KTMF than the selected fit; " + "the earlier target-fit search did not choose it" + ) -@retry(stop=stop_after_delay(30)) -def vsx_auid(ra, dec, radius=0.01, maglimit=14): - try: - url = f"https://www.aavso.org/vsx/index.php?view=api.list&ra={ra}&dec={dec}&radius={radius}&tomag={maglimit}&format=json" - result = requests.get(url) - return result.json()['VSXObjects']['VSXObject'][0]['AUID'] - except Exception: - log.info("\nThe target star does not have an AUID.") - return False + if np.isfinite(candidate_transit_delta_bic) and np.isfinite(selected_transit_delta_bic): + if candidate_transit_delta_bic < selected_transit_delta_bic - 1e-12: + return ( + "not selected: transit-vs-flat Delta BIC was " + f"{candidate_transit_delta_bic:.2f} vs {selected_transit_delta_bic:.2f} for the selected fit" + ) + if candidate_transit_delta_bic > selected_transit_delta_bic + 1e-12: + return ( + "not selected: this post-selection diagnostic fit has stronger transit evidence " + "than the selected fit; the earlier target-fit search did not choose it" + ) + if selected_comp_num is None: + return "not selected: another candidate remained preferred in the target-fit search" + return f"not selected: Comp {selected_comp_num} remained preferred in the target-fit search" -@retry(stop=stop_after_delay(30)) -def vsx_variable(ra, dec, radius=0.01, maglimit=14): - try: - url = f"https://www.aavso.org/vsx/index.php?view=api.list&ra={ra}&dec={dec}&radius={radius}&tomag={maglimit}&format=json" - result = requests.get(url) - var = result.json()['VSXObjects']['VSXObject'][0]['Category'] - if var.lower() == "variable": - vname = result.json()['VSXObjects']['VSXObject'][0]['Name'] - vdec = result.json()['VSXObjects']['VSXObject'][0]['Declination2000'] - vra = result.json()['VSXObjects']['VSXObject'][0]['RA2000'] - log_info(f"\nVSX variable check found {vname} at RA {vra}, DEC {vdec}\n" - f"and will be removed from reduction.", warn=True) - return True - return False - except Exception: - return False +def format_fit_parameter_with_uncertainty(value, error=None, scale=1.0, suffix=""): + if value is None or not np.isfinite(value): + return "n/a" -def build_comp_ra_dec(ra_wcs, dec_wcs, comp_stars): - comp_ra_dec = [] - for _, comp_star in enumerate(comp_stars[:]): - comp_ra_dec.append([ra_wcs[int(comp_star[1])][int(comp_star[0])], - dec_wcs[int(comp_star[1])][int(comp_star[0])]]) - return comp_ra_dec + scaled_value = float(value) * scale + if error is None or not np.isfinite(error) or error < 0: + return f"{round_to_2(scaled_value)}{suffix}" -def check_for_variable_stars(ra_wcs, dec_wcs, comp_stars): - for i, comp_star in enumerate(comp_stars[:]): - ra = ra_wcs[int(comp_star[1])][int(comp_star[0])] - dec = dec_wcs[int(comp_star[1])][int(comp_star[0])] + scaled_error = float(error) * abs(scale) + return f"{format_value_with_uncertainty(scaled_value, scaled_error)}{suffix}" - log_info(f"\nChecking for variability in Comparison Star #{i + 1}:" - f"\n\tPixel X: {comp_star[0]} Pixel Y: {comp_star[1]}") - if query_variable_star_apis(ra, dec): - comp_stars.remove(comp_star) +def summarize_lightcurve_fit_parameters(fit): + if fit is None or not hasattr(fit, 'parameters'): + return None -@retry(stop=stop_after_delay(30)) -def gaia_query(sample, radius): - try: - gaia_query = Gaia.cone_search(sample, radius) - return gaia_query.get_results() - except Exception: - return False + parameters = getattr(fit, 'parameters', {}) or {} + errors = getattr(fit, 'errors', {}) or {} + fit_method = getattr(fit, 'ns_type', 'lm') + summary_parts = [ + f"fit_method={fit_method}", + f"Tmid={format_fit_parameter_with_uncertainty(parameters.get('tmid'), errors.get('tmid'))}", + f"Rp/R*={format_fit_parameter_with_uncertainty(parameters.get('rprs'), errors.get('rprs'))}", + ] + + depth_summary = fit_transit_depth_summary(fit) + summary_parts.append( + "area_depth=" + f"{format_fit_parameter_with_uncertainty(depth_summary.get('area_depth'), depth_summary.get('area_depth_error'), suffix='%')}" + ) + summary_parts.append( + "observable_depth=" + f"{format_fit_parameter_with_uncertainty(depth_summary.get('observable_depth'), depth_summary.get('observable_depth_error'), suffix='%')}" + ) + summary_parts.append(f"inc={format_fit_parameter_with_uncertainty(parameters.get('inc'), errors.get('inc'))}") + if getattr(fit, 'airmass_fit_skipped', False): + summary_parts.append("airmass=skipped") + else: + baseline_key = 'a0' if 'a0' in parameters else 'a1' + summary_parts.append( + f"{baseline_key}={format_fit_parameter_with_uncertainty(parameters.get(baseline_key), errors.get(baseline_key))}" + ) + summary_parts.append( + f"a2={format_fit_parameter_with_uncertainty(parameters.get('a2'), errors.get('a2'))}" + ) -@retry(stop=stop_after_delay(30)) -def simbad_query(sample): - try: - simbad_result = Simbad.query_region(sample, radius=20 * u.arcsec) - return simbad_result['MAIN_ID'][0].decode("utf-8") - except Exception: - return False + return ", ".join(summary_parts) -# Apply calibrations if applicable -def apply_cals(image_data, gen_dark, gen_bias, gen_flat, i): - if gen_dark is not None and gen_dark.size != 0: - if i == 0: - log_info("Dark subtracting images.") - image_data = image_data - gen_dark - elif gen_bias is not None and gen_bias.size != 0: # if a dark is not available, then at least subtract off the pedestal via the bias - if i == 0: - log_info("Bias-correcting images.") - image_data = image_data - gen_bias - else: - pass +def log_comparison_candidate_fit_summaries(candidate_fit_summaries, photometry_info): + if not candidate_fit_summaries: + return - if gen_flat is not None and gen_flat.size != 0: - if i == 0: - log_info("Flattening images.") - gen_flat[gen_flat == 0] = 1 - image_data = image_data / gen_flat - return image_data + selection_basis = photometry_info.get('selection_basis', 'target_fit').replace('_', '-') + log_info("\nComparison-star lightcurve fit diagnostics:") + log_info(f"Selection basis: {selection_basis}") + log_info( + "Selection metric: " + f"{comparison_selection_metric_label(photometry_info.get('selection_metric', 'ktmf'))}" + ) -def calculate_demosaic_mult(demosaic_out): - if not demosaic_out: - return None - # Build vector to convert RBG pixels to single output - if isinstance(demosaic_out, list): - demosaic_mult = np.array(demosaic_out) - elif demosaic_out == 'red': - demosaic_mult = np.array([ 1.0, 0.0, 0.0 ]) - elif demosaic_out == 'green': - demosaic_mult = np.array([ 0.0, 1.0, 0.0 ]) - elif demosaic_out == 'blue': - demosaic_mult = np.array([ 0.0, 0.0, 1.0 ]) - elif demosaic_out == 'gray': - demosaic_mult = np.array([ 0.299, 0.587, 0.114 ]) # Same as rbg2gray - elif demosaic_out == 'blueblock': - demosaic_mult = np.array([ 0.299, 0.587, 0.0 ]) # drop blue, same mix of red, green as gray - else: # Green default - demosaic_mult = np.array([ 0.0, 1.0, 0.0 ]) - # Normalize - demosaic_mult = demosaic_mult / (demosaic_mult[0]+demosaic_mult[1]+demosaic_mult[2]) - return demosaic_mult + for summary in candidate_fit_summaries: + selected_label = " [selected]" if summary.get('selected') else "" + position_text = format_comp_star_position(summary.get('position')) + diagnostics = summary.get('fit_diagnostics') or {} + usable_point_count = diagnostics.get('usable_point_count', 0) + coverage_text = format_comp_star_coverage_text(summary) + eebls_text = format_eebls_snr(summary.get('eebls_snr', np.nan)) + transit_delta_bic_text = format_transit_delta_bic(summary.get('transit_delta_bic', np.nan)) + residual_text = format_residual_scatter(summary.get('residual_scatter', np.nan)) + ktmf_text = format_ktmf_metric(summary.get('ktmf_metric', np.nan)) + reason_text = comparison_candidate_fit_selection_reason(summary, photometry_info) + log_info( + f" {summary['label']}{selected_label} ({position_text}): " + f"coverage={coverage_text}, " + f"usable_after_filters={usable_point_count}, fit_points={summary['fit_point_count']}, " + f"eebls_snr={eebls_text}, transit_delta_bic={transit_delta_bic_text}, " + f"residual_scatter={residual_text}, ktmf={ktmf_text}, reason={reason_text}" + ) + parameter_summary = summary.get('parameter_summary') + if parameter_summary: + log_info(f" parameters: {parameter_summary}") + for contribution in summary.get('ktmf_contributions', []): + log_info(f" {format_ktmf_contribution(contribution)}") + + +def fit_lightcurve_to_every_comparison_candidate(times, jd_times, airmass, ld, p_dict, comp_stars, + psf_data, aper_data, photometry_info, + plot_time_range=None, + disable_vertical_flux_normalization=False, + skip_low_comparison_coverage_rejection=False, + use_impactparameter_rather_than_inclination_to_fit=True, + use_eebls_to_initialize_tmid_and_bounds=True, + psf_flux_data=None, + psf_noise_data=None, + exposure_times_seconds=None, + gain_e_per_adu=None): + if photometry_info.get('best_fit_lc') is None or not comp_stars: + return [] -# If demosaic requested, process -def demosaic_img(image_data, demosaic_fmt, demosaic_out, demosaic_mult, i): - if demosaic_fmt: - if i == 0: - log_info(f"Demosaicing images (mapping {demosaic_fmt} to {demosaic_out})") - img_dtype = image_data.dtype # Save data type - new_image_data = demosaicing_CFA_Bayer_bilinear(image_data, demosaic_fmt) - image_data = (new_image_data @ demosaic_mult).astype(img_dtype) - return image_data + use_psf_photometry = photometry_info.get('min_aperture') == 0 + if use_psf_photometry: + frame_count = psf_data['target'].shape[0] + else: + frame_count = aper_data['target'].shape[0] + if use_psf_photometry: + psf_flux_data = psf_flux_data_source(psf_data, psf_flux_data) + target_flux = psf_flux_series_from_rows(psf_flux_data['target']) + target_flux_error = ( + np.asarray(psf_noise_data.get('target'), dtype=float) + if isinstance(psf_noise_data, dict) and 'target' in psf_noise_data + else None + ) + comp_flux_map = { + f"comp{comp_index + 1}": psf_flux_series_from_rows( + psf_flux_data[f"comp{comp_index + 1}"], + psf_quality_mask_for_key( + psf_data, + f"comp{comp_index + 1}", + frame_count, + psf_flux_data=psf_flux_data, + ), + ) + for comp_index in range(len(comp_stars)) + } + comp_error_map = { + f"comp{comp_index + 1}": mask_series_with_quality( + psf_noise_data[f"comp{comp_index + 1}"], + psf_quality_mask_for_key( + psf_data, + f"comp{comp_index + 1}", + frame_count, + psf_flux_data=psf_flux_data, + ), + ) + for comp_index in range(len(comp_stars)) + if isinstance(psf_noise_data, dict) and f"comp{comp_index + 1}" in psf_noise_data + } + else: + aperture_index = photometry_info.get('aperture_index') + annulus_index = photometry_info.get('annulus_index') + if aperture_index is None or annulus_index is None: + return [] + target_flux = np.asarray(aper_data['target'][:, aperture_index, annulus_index], dtype=float) + target_flux_error = ( + np.asarray(aper_data['target_unc'][:, aperture_index, annulus_index], dtype=float) + if 'target_unc' in aper_data + else None + ) + comp_flux_map = { + f"comp{comp_index + 1}": mask_series_with_quality( + aper_data[f"comp{comp_index + 1}"][:, aperture_index, annulus_index], + psf_quality_mask_for_key(psf_data, f"comp{comp_index + 1}", frame_count), + ) + for comp_index in range(len(comp_stars)) + } + comp_error_map = { + f"comp{comp_index + 1}": mask_series_with_quality( + aper_data[f"comp{comp_index + 1}_unc"][:, aperture_index, annulus_index], + psf_quality_mask_for_key(psf_data, f"comp{comp_index + 1}", frame_count), + ) + for comp_index in range(len(comp_stars)) + if f"comp{comp_index + 1}_unc" in aper_data + } -def vsp_query(file, axis, obs_filter, img_scale, maglimit=14, user_comp_stars=None, user_targ_star=None): - if user_comp_stars is None: - user_comp_stars = [] + candidate_fit_summaries = [] + selected_comp_star_num = photometry_info.get('comp_star_num') + coverage_summary = comparison_star_coverage_summary( + comp_flux_map, + skip_rejection=skip_low_comparison_coverage_rejection, + validity_mask_func=( + robust_flux_floor_mask if use_psf_photometry else valid_comparison_frame_mask + ), + ) - vsp_comp_stars_info = {} - vsp_star_count = 0 + for comp_index, position in enumerate(comp_stars): + label = f"Comp {comp_index + 1}" + ckey = f"comp{comp_index + 1}" + comp_flux_series = comp_flux_map[ckey] - # Build combined list for comps and target - there are known cases when AAVsO comps have planets (XO-2 N) - # Plus, we don't want comp too close to target - targ_and_comp_stars = user_comp_stars[:] - if user_targ_star is not None: - targ_and_comp_stars.append(user_targ_star) + if use_psf_photometry: + target_shape_mask = target_psf_shape_quality_mask( + target_psf_quality_rows(psf_data, psf_flux_data=psf_flux_data), + psf_quality_rows_for_key(psf_data, ckey, psf_flux_data=psf_flux_data), + ) + if target_shape_mask.shape[0] != frame_count: + target_shape_mask = np.ones(frame_count, dtype=bool) + candidate_target_flux = mask_series_with_quality(target_flux, target_shape_mask) + candidate_target_flux_error = ( + None + if target_flux_error is None + else mask_series_with_quality(target_flux_error, target_shape_mask) + ) + candidate_comp_flux_error = comp_error_map.get(ckey) + fit_mask = target_shape_mask & robust_target_reference_flux_mask(candidate_target_flux, comp_flux_series) + else: + candidate_target_flux = target_flux + candidate_target_flux_error = target_flux_error + candidate_comp_flux_error = comp_error_map.get(ckey) + fit_mask = valid_comparison_frame_mask(candidate_target_flux) & valid_comparison_frame_mask(comp_flux_series) + coverage_count = coverage_summary[ckey]['coverage_count'] + coverage_total_frame_count = coverage_summary[ckey]['coverage_total_frame_count'] + coverage_reference_count = coverage_summary[ckey]['coverage_reference_count'] + coverage_min_required_count = coverage_summary[ckey]['coverage_min_required_count'] + coverage_rejected = coverage_summary[ckey]['coverage_rejected'] + coverage_rejection_detail = format_comp_star_coverage_rejection_detail(coverage_summary[ckey]) + fit_result, target_fit_flux, comp_fit_flux = None, None, None + fit_diagnostics = { + 'input_point_count': int(times.shape[0]), + 'has_reference_flux': True, + 'relative_flux_point_count': 0, + 'sigma_clip_point_count': 0, + 'usable_point_count': 0, + 'failed_stage': 'coverage', + 'failure_reason': ( + f"only {coverage_count} frame(s) had finite positive comparison flux; need at least 2 to fit." + ), + } + if coverage_rejected: + fit_diagnostics['failure_reason'] = ( + "comparison candidate rejected after iterative low-coverage clipping " + f"({coverage_rejection_detail})." + ) + elif coverage_count > 1: + fit_diagnostics = diagnose_lightcurve_fit_inputs( + times[fit_mask], + candidate_target_flux[fit_mask], + comp_flux_series[fit_mask], + airmass[fit_mask], + target_flux_error=None if candidate_target_flux_error is None else candidate_target_flux_error[fit_mask], + comp_flux_error=None if candidate_comp_flux_error is None else candidate_comp_flux_error[fit_mask], + enforce_relative_flux_max=False, + expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), + ) + if not coverage_rejected and coverage_count > 1 and fit_diagnostics['failure_reason'] is None: + fit_result, target_fit_flux, comp_fit_flux = fit_lightcurve( + times[fit_mask], + candidate_target_flux[fit_mask], + comp_flux_series[fit_mask], + airmass[fit_mask], + ld, + p_dict, + jd_times[fit_mask], + target_flux_error=None if candidate_target_flux_error is None else candidate_target_flux_error[fit_mask], + comp_flux_error=None if candidate_comp_flux_error is None else candidate_comp_flux_error[fit_mask], + allow_mid_transit_range_warning=False, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + final_fit_mode='ns', + use_impactparameter_rather_than_inclination_to_fit=use_impactparameter_rather_than_inclination_to_fit, + plot_time_range=plot_time_range, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, + compute_eebls_diagnostics=True, + exposure_times_seconds=( + None + if exposure_times_seconds is None + else np.asarray(exposure_times_seconds, dtype=float)[fit_mask] + ), + gain_e_per_adu=gain_e_per_adu, + ) + fit_diagnostics = ensure_lightcurve_fit_failure_reason( + fit_diagnostics, + fit_result, + failed_stage='nested_fit', + failure_reason="the nested lightcurve fitter did not converge to a usable solution.", + ) - wcs_hdr = search_wcs(file) - fov = (img_scale * max(axis)) / 60 - ra, dec = wcs_hdr.pixel_to_world_values(axis[0] // 2, axis[1] // 2) - # Respect limits from AAVSO API (as reported by API error messages) - if fov > 180 and maglimit > 12: - maglimit = 12 + fit_point_count = 0 if target_fit_flux is None else int(len(target_fit_flux)) + parameter_summary = summarize_lightcurve_fit_parameters(fit_result) + + candidate_fit_summaries.append({ + 'comp_index': comp_index, + 'label': label, + 'position': position, + 'selected': selected_comp_star_num == comp_index + 1, + 'fit': fit_result, + 'eebls_snr': extract_lightcurve_fit_eebls_snr(fit_result), + 'transit_delta_bic': extract_lightcurve_fit_transit_delta_bic(fit_result), + 'residual_scatter': extract_lightcurve_fit_residual_scatter(fit_result), + 'ktmf_metric': extract_lightcurve_fit_ktmf_metric(fit_result), + 'ktmf_contributions': extract_lightcurve_fit_ktmf_contributions(fit_result), + 'coverage_count': coverage_count, + 'coverage_total_frame_count': coverage_total_frame_count, + 'coverage_reference_count': coverage_reference_count, + 'coverage_min_required_count': coverage_min_required_count, + 'coverage_rejected': coverage_rejected, + 'coverage_rejection_threshold_count': coverage_summary[ckey].get('coverage_rejection_threshold_count'), + 'coverage_rejection_reference_count': coverage_summary[ckey].get('coverage_rejection_reference_count'), + 'coverage_rejection_scatter': coverage_summary[ckey].get('coverage_rejection_scatter'), + 'coverage_rejection_iteration': coverage_summary[ckey].get('coverage_rejection_iteration'), + 'fit_point_count': fit_point_count, + 'fit_diagnostics': fit_diagnostics, + 'failure_reason': fit_diagnostics.get('failure_reason'), + 'fit_method': None if fit_result is None else getattr(fit_result, 'ns_type', 'lm'), + 'parameter_summary': parameter_summary, + }) + + return candidate_fit_summaries + + +def normalize_flux_series(flux_values, validity_mask_func=valid_comparison_frame_mask): + flux_values = np.asarray(flux_values, dtype=float) + normalized = np.full(flux_values.shape, np.nan, dtype=float) + finite_mask = validity_mask_func(flux_values) + if np.count_nonzero(finite_mask) < 5: + return normalized + + flux_median = bn.nanmedian(flux_values[finite_mask]) + if not np.isfinite(flux_median) or flux_median <= 0: + return normalized + + normalized[finite_mask] = flux_values[finite_mask] / flux_median + return normalized + + +def normalized_ratio_series(numerator_flux, denominator_flux): + numerator_flux = np.asarray(numerator_flux, dtype=float) + denominator_flux = np.asarray(denominator_flux, dtype=float) + with np.errstate(divide='ignore', invalid='ignore'): + ratio = np.divide(numerator_flux, denominator_flux) + ratio[~np.isfinite(ratio)] = np.nan + ratio[ratio <= 0] = np.nan + return ratio + + +def build_normalized_comp_ensemble(normalized_flux_map, exclude_key): + ensemble_members = [flux for key, flux in normalized_flux_map.items() if key != exclude_key] + if not ensemble_members: + return None - url = f"https://www.aavso.org/apps/vsp/api/chart/?format=json&ra={ra:5f}&dec={dec:5f}&fov={fov}&maglimit={maglimit}" - result = requests.get(url) - data = result.json() - chart_id = data['chartid'] + ensemble_stack = np.vstack(ensemble_members) + valid_mask = np.any(np.isfinite(ensemble_stack), axis=0) + if not np.any(valid_mask): + return None - if obs_filter == "CV": - obs_filter = "V" - elif obs_filter == "R": - obs_filter = "Rc" + ensemble = np.full(ensemble_stack.shape[1], np.nan, dtype=float) + ensemble[valid_mask] = np.nanmedian(ensemble_stack[:, valid_mask], axis=0) + return ensemble - if data['photometry']: - for star in data['photometry']: - ra_deg, dec_deg = radec_hours_to_degree(star['ra'], star['dec']) - ra_pix, dec_pix = wcs_hdr.world_to_pixel_values(ra_deg, dec_deg) - if (ra_pix < axis[0] and dec_pix < axis[1]) and (ra_pix > 1 and dec_pix > 1): - vsp_star = [int(ra_pix.min()), int(dec_pix.min())] - exist, vsp_star = check_comp_star_exists(targ_and_comp_stars, vsp_star) +def build_absolute_comp_ensemble_flux(comp_flux_map, active_keys, + validity_mask_func=valid_comparison_frame_mask): + normalized_members = [] + member_medians = [] + member_keys = [] + for key in active_keys: + if key not in comp_flux_map: + continue + flux_values = np.asarray(comp_flux_map[key], dtype=float) + valid_mask = validity_mask_func(flux_values) + if np.count_nonzero(valid_mask) < 5: + continue + member_median = float(bn.nanmedian(flux_values[valid_mask])) + if not np.isfinite(member_median) or member_median <= 0: + continue + normalized_flux = np.full(flux_values.shape, np.nan, dtype=float) + normalized_flux[valid_mask] = flux_values[valid_mask] / member_median + normalized_members.append(normalized_flux) + member_medians.append(member_median) + member_keys.append(key) + + if not normalized_members: + return None, [] + + ensemble_stack = np.vstack(normalized_members) + valid_mask = np.any(np.isfinite(ensemble_stack), axis=0) + ensemble = np.full(ensemble_stack.shape[1], np.nan, dtype=float) + ensemble[valid_mask] = np.nanmedian(ensemble_stack[:, valid_mask], axis=0) + scale = float(np.nanmedian(member_medians)) + if not np.isfinite(scale) or scale <= 0: + scale = 1.0 + return ensemble * scale, member_keys + + +def build_absolute_comp_ensemble_uncertainty(comp_flux_map, comp_error_map, member_keys, + validity_mask_func=valid_comparison_frame_mask): + if not member_keys or not isinstance(comp_error_map, dict): + return None + + normalized_variances = [] + member_medians = [] + for key in member_keys: + if key not in comp_flux_map or key not in comp_error_map: + continue + flux_values = np.asarray(comp_flux_map[key], dtype=float) + error_values = np.asarray(comp_error_map[key], dtype=float) + if flux_values.shape != error_values.shape: + continue + valid_flux_mask = validity_mask_func(flux_values) + if np.count_nonzero(valid_flux_mask) < 5: + continue + member_median = float(bn.nanmedian(flux_values[valid_flux_mask])) + if not np.isfinite(member_median) or member_median <= 0: + continue + valid_error = valid_flux_mask & np.isfinite(error_values) & (error_values > 0) + normalized_variance = np.full(flux_values.shape, np.nan, dtype=float) + normalized_variance[valid_error] = (error_values[valid_error] / member_median) ** 2 + normalized_variances.append(normalized_variance) + member_medians.append(member_median) - if obs_filter in [band['band'] for band in star['bands']]: - star_info = next(band for band in star['bands'] if band['band'] == obs_filter) + if not normalized_variances: + return None - vsp_comp_stars_info[star['auid']] = { - 'pos': vsp_star, - 'mag': star_info['mag'], - 'error': star_info['error'] - } + variance_stack = np.vstack(normalized_variances) + valid_count = np.count_nonzero(np.isfinite(variance_stack), axis=0) + summed_variance = np.nansum(variance_stack, axis=0) + ensemble_variance = np.full(variance_stack.shape[1], np.nan, dtype=float) + valid_frames = valid_count > 0 + ensemble_variance[valid_frames] = summed_variance[valid_frames] / (valid_count[valid_frames] ** 2) + scale = float(np.nanmedian(member_medians)) + if not np.isfinite(scale) or scale <= 0: + scale = 1.0 + ensemble_unc = np.full(variance_stack.shape[1], np.nan, dtype=float) + finite_var = np.isfinite(ensemble_variance) & (ensemble_variance >= 0) + ensemble_unc[finite_var] = np.sqrt(ensemble_variance[finite_var]) * scale + return ensemble_unc + + +def build_relative_comparison_ensemble_series(target_flux, target_flux_error, + comp_flux_map, comp_error_map, member_keys, + validity_mask_func=valid_comparison_frame_mask): + """Combine exactly the requested reference members without catalogue magnitudes.""" + requested_keys = [key for key in member_keys or [] if key] + ensemble_flux, used_keys = build_absolute_comp_ensemble_flux( + comp_flux_map, + requested_keys, + validity_mask_func=validity_mask_func, + ) + if ensemble_flux is None or used_keys != requested_keys: + return { + 'applied': False, + 'failure_reason': 'not every requested comparison had a usable flux series', + 'member_keys': used_keys, + } + ensemble_error = build_absolute_comp_ensemble_uncertainty( + comp_flux_map, + comp_error_map, + used_keys, + validity_mask_func=validity_mask_func, + ) - if not exist: - vsp_star_count = add_vsp_star(vsp_star_count, user_comp_stars, vsp_star) + target_flux = np.asarray(target_flux, dtype=float) + if target_flux_error is None: + target_flux_error = source_flux_uncertainty_from_counts(target_flux) + target_flux_error = np.asarray(target_flux_error, dtype=float) + if target_flux_error.shape != target_flux.shape: + target_flux_error = source_flux_uncertainty_from_counts(target_flux) + if ensemble_error is None or np.asarray(ensemble_error).shape != ensemble_flux.shape: + ensemble_error = source_flux_uncertainty_from_counts(ensemble_flux) + ensemble_error = np.asarray(ensemble_error, dtype=float) + + all_members_valid = np.ones(target_flux.shape, dtype=bool) + for key in requested_keys: + member_flux = np.asarray(comp_flux_map[key], dtype=float) + if member_flux.shape != target_flux.shape: + return { + 'applied': False, + 'failure_reason': f'{key} had a mismatched flux-series shape', + 'member_keys': used_keys, + } + all_members_valid &= validity_mask_func(member_flux) + + valid = ( + all_members_valid + & np.isfinite(target_flux) + & (target_flux > 0) + & np.isfinite(ensemble_flux) + & (ensemble_flux > 0) + & np.isfinite(target_flux_error) + & (target_flux_error >= 0) + & np.isfinite(ensemble_error) + & (ensemble_error >= 0) + ) + relative_flux = np.full(target_flux.shape, np.nan, dtype=float) + relative_flux_error = np.full(target_flux.shape, np.nan, dtype=float) + with np.errstate(divide='ignore', invalid='ignore'): + relative_flux[valid] = target_flux[valid] / ensemble_flux[valid] + relative_flux_error[valid] = relative_flux[valid] * np.sqrt( + (target_flux_error[valid] / target_flux[valid]) ** 2 + + (ensemble_error[valid] / ensemble_flux[valid]) ** 2 + ) + valid &= np.isfinite(relative_flux) & (relative_flux > 0) + valid &= np.isfinite(relative_flux_error) & (relative_flux_error > 0) + return { + 'applied': bool(np.count_nonzero(valid) >= LIGHTCURVE_MIN_VALID_POINTS), + 'failure_reason': ( + None + if np.count_nonzero(valid) >= LIGHTCURVE_MIN_VALID_POINTS + else 'fewer than five frames contained every requested comparison member' + ), + 'member_keys': used_keys, + 'reference_flux': ensemble_flux, + 'reference_flux_error': ensemble_error, + 'relative_flux': relative_flux, + 'relative_flux_error': relative_flux_error, + 'valid_mask': valid, + } - if len(vsp_comp_stars_info) > 1: - break - if not vsp_star_count: - log_info("\nNo comparison stars were gathered from AAVSO.\n") +def comparison_star_coverage_summary(comp_flux_map, + min_fraction=COMPARISON_STAR_MIN_COVERAGE_FRACTION, + min_points=COMPARISON_STAR_MIN_VALID_FRAMES, + skip_rejection=False, + validity_mask_func=valid_comparison_frame_mask): + comp_keys = list(comp_flux_map.keys()) + if not comp_keys: + return {} - return vsp_comp_stars_info, chart_id + coverage_counts = { + key: int(np.count_nonzero(validity_mask_func(comp_flux_map[key]))) + for key in comp_keys + } + total_frame_count = max(np.asarray(comp_flux_map[key]).shape[0] for key in comp_keys) + effective_min_points = int(min_points) if total_frame_count >= int(min_points) else 0 + active_keys = list(comp_keys) + coverage_reference_count = float(np.nanmedian([coverage_counts[key] for key in active_keys])) + coverage_min_required_count = max(effective_min_points, 0) + coverage_scatter = np.nan + coverage_rejection_info = {} + + for iteration_index in range(COMPARISON_STAR_COVERAGE_MAX_ITERS): + active_counts = np.asarray([coverage_counts[key] for key in active_keys], dtype=float) + if active_counts.size == 0: + break + coverage_reference_count = float(np.nanmedian(active_counts)) + coverage_scatter = robust_scatter(active_counts) + # Coverage is only an availability gate. High-scatter comparison stars + # are handled by the suitability outlier pass after coverage-qualified + # stars have been scored. + coverage_min_required_count = max( + effective_min_points, + int(np.ceil(float(min_fraction) * coverage_reference_count)), + ) -def add_vsp_star(vsp_star_count, user_comp_stars, vsp_star): - user_comp_stars.append(vsp_star) - log_info(f"\nAdded Comparison Star #{len(user_comp_stars)}, coordinates {vsp_star} from AAVSO") + kept_keys = [key for key in active_keys if coverage_counts[key] >= coverage_min_required_count] + if len(kept_keys) == len(active_keys): + break + kept_key_set = set(kept_keys) + for key in active_keys: + if key in kept_key_set or key in coverage_rejection_info: + continue + coverage_rejection_info[key] = { + 'coverage_rejection_threshold_count': coverage_min_required_count, + 'coverage_rejection_reference_count': coverage_reference_count, + 'coverage_rejection_scatter': coverage_scatter, + 'coverage_rejection_iteration': iteration_index + 1, + } + active_keys = kept_keys + + coverage_summary = {} + active_key_set = set(active_keys) + for key in comp_keys: + rejection_info = coverage_rejection_info.get(key, {}) + coverage_summary[key] = { + 'coverage_count': coverage_counts[key], + 'coverage_total_frame_count': total_frame_count, + 'coverage_reference_count': coverage_reference_count, + 'coverage_median_count': coverage_reference_count, + 'coverage_scatter': coverage_scatter, + 'coverage_min_required_count': coverage_min_required_count, + 'coverage_rejected': False if skip_rejection else key not in active_key_set, + 'coverage_rejection_threshold_count': rejection_info.get('coverage_rejection_threshold_count'), + 'coverage_rejection_reference_count': rejection_info.get('coverage_rejection_reference_count'), + 'coverage_rejection_scatter': rejection_info.get('coverage_rejection_scatter'), + 'coverage_rejection_iteration': rejection_info.get('coverage_rejection_iteration'), + } - return vsp_star_count + 1 + return coverage_summary + + +def apply_comparison_star_suitability_outlier_rejection( + comp_summaries, + sigma=COMPARISON_STAR_SUITABILITY_OUTLIER_SIGMA, + min_candidates=COMPARISON_STAR_SUITABILITY_MIN_CANDIDATES, + eligible_indices=None, +): + if eligible_indices is None: + eligible_indices = [ + index + for index, summary in enumerate(comp_summaries) + if ( + not summary.get('coverage_rejected') + and np.isfinite(summary.get('aggregate_score', np.inf)) + ) + ] + else: + eligible_indices = [ + int(index) + for index in eligible_indices + if ( + 0 <= int(index) < len(comp_summaries) + and not comp_summaries[int(index)].get('coverage_rejected') + and np.isfinite(comp_summaries[int(index)].get('aggregate_score', np.inf)) + ) + ] + clipping_candidate_floor = max(3, int(min_candidates)) + reference_score = np.nan + scatter = np.nan + high_threshold = np.nan + kept_indices = list(eligible_indices) + rejected_index_set = set() + + if len(eligible_indices) >= clipping_candidate_floor: + eligible_scores = np.asarray( + [comp_summaries[index]['aggregate_score'] for index in eligible_indices], + dtype=float, + ) + if eligible_scores.size and np.any(np.isfinite(eligible_scores)): + reference_score = float(np.nanmedian(eligible_scores)) + scatter = robust_scatter(eligible_scores) + if np.isfinite(scatter) and scatter > 0: + high_threshold = reference_score + float(sigma) * scatter + kept_indices = [ + index for index in eligible_indices + if comp_summaries[index]['aggregate_score'] <= high_threshold + ] + rejected_index_set = set(eligible_indices) - set(kept_indices) + + for index, summary in enumerate(comp_summaries): + summary['suitability_outlier_rejected'] = index in rejected_index_set + summary['suitability_reference_score'] = reference_score + summary['suitability_scatter'] = scatter + summary['suitability_high_threshold'] = high_threshold + + return { + 'eligible_indices': eligible_indices, + 'active_indices': kept_indices, + 'rejected_indices': sorted(rejected_index_set), + 'reference_score': reference_score, + 'scatter': scatter, + 'high_threshold': high_threshold, + } -def check_comp_star_exists(user_stars, vsp_star, tol=10): - """Checks if a comparison star from VSP exists in the user-entered - comparison star list +def comparison_star_image_outlier_summary( + normalized_flux_map, + active_keys, + sigma=COMPARISON_IMAGE_OUTLIER_SIGMA, + min_active_stars=COMPARISON_IMAGE_OUTLIER_MIN_ACTIVE_STARS, + min_valid_pairs=COMPARISON_IMAGE_OUTLIER_MIN_VALID_PAIRS, +): + active_keys = [key for key in active_keys if key in normalized_flux_map] + series_length = 0 + for key in active_keys: + flux_values = np.asarray(normalized_flux_map.get(key), dtype=float) + if flux_values.ndim == 1: + series_length = flux_values.shape[0] + break - Parameters - ---------- - user_stars : list - A header file that may include the airmass or altitude from when the image was taken - vsp_star : list - Right Ascension - tol : float - Declination + keep_mask = np.ones(series_length, dtype=bool) + summary = { + 'frame_keep_mask': keep_mask, + 'rejected_frame_indices': [], + 'rejected_frame_count': 0, + 'valid_pair_counts': np.zeros(series_length, dtype=int), + 'outlier_pair_counts': np.zeros(series_length, dtype=int), + 'available_pair_count': 0, + 'required_valid_pair_count': 0, + 'sigma': float(sigma), + } - Returns - ------- - bool - True if VSP star exists in user entered stars, otherwise False - list - Pixel coordinate of either the user entered star (exists), otherwise pixel coordinates - of VSP - """ - for user_star in user_stars: - pixel_distance = [abs(star1 - star2) for star1, star2 in zip(user_star, vsp_star)] + if series_length == 0 or len(active_keys) < max(2, int(min_active_stars)): + return summary - if all(i <= tol for i in pixel_distance): - return True, user_star - return False, vsp_star + pairwise_valid_flags = [] + pairwise_outlier_flags = [] + for index, key in enumerate(active_keys): + numerator_flux = np.asarray(normalized_flux_map[key], dtype=float) + if numerator_flux.ndim != 1 or numerator_flux.shape[0] != series_length: + continue + for other_key in active_keys[index + 1:]: + denominator_flux = np.asarray(normalized_flux_map[other_key], dtype=float) + if denominator_flux.ndim != 1 or denominator_flux.shape[0] != series_length: + continue -# Aligns imaging data from .fits file to easily track the host and comparison star's positions -def transformation(image_data, file_name, roi=1): - # crop image to ROI - height = image_data.shape[1] - width = image_data.shape[2] - roix = slice(int(width * (0.5 - roi / 2)), int(width * (0.5 + roi / 2))) - roiy = slice(int(height * (0.5 - roi / 2)), int(height * (0.5 + roi / 2))) + ratio = normalized_ratio_series(numerator_flux, denominator_flux) + valid_mask = np.isfinite(ratio) + if np.count_nonzero(valid_mask) < LIGHTCURVE_MIN_VALID_POINTS: + continue + + center, scatter = sigma_clipped_nanmedian(ratio[valid_mask], sigma=4.0, max_iters=3) + if not np.isfinite(center): + center = float(bn.nanmedian(ratio[valid_mask])) + robust_pair_scatter = robust_scatter(ratio[valid_mask] - center) + if np.isfinite(robust_pair_scatter) and robust_pair_scatter > 0: + scatter = robust_pair_scatter + if not np.isfinite(scatter) or scatter <= 0: + scatter = robust_scatter(ratio[valid_mask] - center) + if not np.isfinite(scatter) or scatter <= 0: + scatter = COMPARISON_IMAGE_OUTLIER_MIN_SCATTER + + outlier_mask = valid_mask & np.greater(np.abs(ratio - center), float(sigma) * scatter) + pairwise_valid_flags.append(valid_mask) + pairwise_outlier_flags.append(outlier_mask) + + available_pair_count = len(pairwise_valid_flags) + required_valid_pair_count = max(int(min_valid_pairs), len(active_keys) - 1) + summary['available_pair_count'] = available_pair_count + summary['required_valid_pair_count'] = required_valid_pair_count + if available_pair_count < required_valid_pair_count: + return summary + + valid_pair_counts = np.sum(np.vstack(pairwise_valid_flags), axis=0).astype(int) + outlier_pair_counts = np.sum(np.vstack(pairwise_outlier_flags), axis=0).astype(int) + rejected_mask = ( + (valid_pair_counts >= required_valid_pair_count) + & (outlier_pair_counts == valid_pair_counts) + & (valid_pair_counts > 0) + ) + keep_mask = ~rejected_mask + + summary.update({ + 'frame_keep_mask': keep_mask, + 'rejected_frame_indices': np.flatnonzero(rejected_mask).astype(int).tolist(), + 'rejected_frame_count': int(np.count_nonzero(rejected_mask)), + 'valid_pair_counts': valid_pair_counts, + 'outlier_pair_counts': outlier_pair_counts, + }) + return summary + + +def comparison_pairwise_log_ratio_outlier_flags( + ratio, + sigma=COMPARISON_IMAGE_OUTLIER_SIGMA, + min_points=LIGHTCURVE_MIN_VALID_POINTS, + scatter_floor=COMPARISON_IMAGE_OUTLIER_MIN_SCATTER, +): + ratio = np.asarray(ratio, dtype=float).reshape(-1) + valid_mask = np.isfinite(ratio) & (ratio > 0) + outlier_mask = np.zeros(ratio.shape, dtype=bool) + direction = np.zeros(ratio.shape, dtype=int) + + if np.count_nonzero(valid_mask) < max(int(min_points), 3): + return valid_mask, outlier_mask, direction - # Find transformation from .FITS files and catch exceptions if not able to. try: - results = aa.find_transform(image_data[1][roiy, roix], image_data[0][roiy, roix]) - return results[0] - except Exception: - ws = 5 - # smooth image and try to align again - windows = view_as_windows(image_data[0], (ws,ws), step=1) - medimg = np.median(windows, axis=(2,3)) + sigma = float(sigma) + except (TypeError, ValueError): + sigma = COMPARISON_IMAGE_OUTLIER_SIGMA + if not np.isfinite(sigma) or sigma <= 0: + sigma = COMPARISON_IMAGE_OUTLIER_SIGMA + + valid_indices = np.flatnonzero(valid_mask) + log_ratio = np.log(ratio[valid_indices]) + center, _ = sigma_clipped_nanmedian(log_ratio, sigma=4.0, max_iters=3) + if not np.isfinite(center): + center = bn.nanmedian(log_ratio) + if not np.isfinite(center): + return valid_mask, outlier_mask, direction + + residuals = log_ratio - center + finite_residuals = residuals[np.isfinite(residuals)] + residual_center = bn.nanmedian(finite_residuals) if finite_residuals.size else np.nan + mad = bn.nanmedian(np.abs(finite_residuals - residual_center)) if finite_residuals.size else np.nan + scatter = 1.4826 * mad if np.isfinite(mad) and mad > 0 else np.nan + if np.isfinite(scatter_floor) and scatter_floor > 0: + if not np.isfinite(scatter) or scatter <= 0: + scatter = float(scatter_floor) + else: + scatter = max(float(scatter), float(scatter_floor)) + if not np.isfinite(scatter) or scatter <= 0: + return valid_mask, outlier_mask, direction + + pair_outliers = np.abs(residuals) > sigma * scatter + outlier_indices = valid_indices[pair_outliers] + outlier_mask[outlier_indices] = True + direction[outlier_indices] = np.sign(residuals[pair_outliers]).astype(int) + return valid_mask, outlier_mask, direction + + +def comparison_star_candidate_frame_outlier_summary( + normalized_flux_map, + candidate_key, + active_keys, + field_image_keep_mask=None, + sigma=COMPARISON_IMAGE_OUTLIER_SIGMA, + min_valid_pairs=COMPARISON_CANDIDATE_FRAME_OUTLIER_MIN_VALID_PAIRS, +): + active_keys = [key for key in active_keys if key in normalized_flux_map] + series_length = 0 + if candidate_key in normalized_flux_map: + candidate_flux = np.asarray(normalized_flux_map[candidate_key], dtype=float) + if candidate_flux.ndim == 1: + series_length = candidate_flux.shape[0] + else: + candidate_flux = np.asarray([], dtype=float) + else: + candidate_flux = np.asarray([], dtype=float) + + keep_mask = np.ones(series_length, dtype=bool) + summary = { + 'frame_keep_mask': keep_mask, + 'rejected_frame_indices': [], + 'rejected_frame_count': 0, + 'valid_pair_counts': np.zeros(series_length, dtype=int), + 'outlier_pair_counts': np.zeros(series_length, dtype=int), + 'positive_outlier_pair_counts': np.zeros(series_length, dtype=int), + 'negative_outlier_pair_counts': np.zeros(series_length, dtype=int), + 'available_pair_count': 0, + 'required_valid_pair_count': max(int(min_valid_pairs), 1), + 'sigma': float(sigma), + } - windows = view_as_windows(image_data[1], (ws,ws), step=1) - medimg1 = np.median(windows, axis=(2,3)) + if series_length == 0 or candidate_key not in active_keys: + return summary - try: - results = aa.find_transform(medimg1[roiy, roix], medimg[roiy, roix]) - return results[0] - except Exception: - pass + if field_image_keep_mask is None: + field_image_keep_mask = np.ones(series_length, dtype=bool) + else: + field_image_keep_mask = np.asarray(field_image_keep_mask, dtype=bool).reshape(-1) + if field_image_keep_mask.shape[0] != series_length: + field_image_keep_mask = np.ones(series_length, dtype=bool) + + peer_keys = [key for key in active_keys if key != candidate_key] + required_valid_pair_count = max(int(min_valid_pairs), 1) + summary['required_valid_pair_count'] = required_valid_pair_count + if len(peer_keys) < required_valid_pair_count: + return summary + + pairwise_valid_flags = [] + pairwise_outlier_flags = [] + pairwise_positive_flags = [] + pairwise_negative_flags = [] + + for peer_key in peer_keys: + peer_flux = np.asarray(normalized_flux_map.get(peer_key), dtype=float) + if peer_flux.ndim != 1 or peer_flux.shape[0] != series_length: + continue - for p in [99, 98, 95, 90]: - for it in [2, 1, 0]: + ratio = normalized_ratio_series(candidate_flux, peer_flux) + ratio[~field_image_keep_mask] = np.nan + valid_mask, outlier_mask, direction = comparison_pairwise_log_ratio_outlier_flags( + ratio, + sigma=sigma, + ) + valid_mask &= field_image_keep_mask + outlier_mask &= valid_mask + if np.count_nonzero(valid_mask) < LIGHTCURVE_MIN_VALID_POINTS: + continue - # create binary mask to align image - mask1 = image_data[1][roiy, roix] > np.percentile(image_data[1][roiy, roix], p) - mask1 = binary_erosion(mask1, iterations=it) + pairwise_valid_flags.append(valid_mask) + pairwise_outlier_flags.append(outlier_mask) + pairwise_positive_flags.append(outlier_mask & (direction > 0)) + pairwise_negative_flags.append(outlier_mask & (direction < 0)) + + available_pair_count = len(pairwise_valid_flags) + summary['available_pair_count'] = available_pair_count + if available_pair_count < required_valid_pair_count: + return summary + + valid_pair_counts = np.sum(np.vstack(pairwise_valid_flags), axis=0).astype(int) + outlier_pair_counts = np.sum(np.vstack(pairwise_outlier_flags), axis=0).astype(int) + positive_outlier_pair_counts = np.sum(np.vstack(pairwise_positive_flags), axis=0).astype(int) + negative_outlier_pair_counts = np.sum(np.vstack(pairwise_negative_flags), axis=0).astype(int) + directional_outlier_counts = np.maximum(positive_outlier_pair_counts, negative_outlier_pair_counts) + rejected_mask = ( + field_image_keep_mask + & (valid_pair_counts >= required_valid_pair_count) + & (directional_outlier_counts >= required_valid_pair_count) + & (directional_outlier_counts > (valid_pair_counts / 2.0)) + ) + keep_mask = ~rejected_mask + + summary.update({ + 'frame_keep_mask': keep_mask, + 'rejected_frame_indices': np.flatnonzero(rejected_mask).astype(int).tolist(), + 'rejected_frame_count': int(np.count_nonzero(rejected_mask)), + 'valid_pair_counts': valid_pair_counts, + 'outlier_pair_counts': outlier_pair_counts, + 'positive_outlier_pair_counts': positive_outlier_pair_counts, + 'negative_outlier_pair_counts': negative_outlier_pair_counts, + }) + return summary + + +def comparison_star_stability_summary(comp_flux_map, airmass, skip_low_coverage_rejection=False, + validity_mask_func=valid_comparison_frame_mask, + bypass_vetting=False): + if not comp_flux_map: + return { + 'pairwise_matrix': np.empty((0, 0), dtype=float), + 'comp_summaries': [], + 'field_score': np.inf, + 'best_comp_index': None, + 'best_comp_score': np.inf, + 'suitability_outlier_rejected_count': 0, + 'suitability_high_threshold': np.nan, + 'suitability_reference_score': np.nan, + 'suitability_scatter': np.nan, + 'field_image_keep_mask': np.ones(0, dtype=bool), + 'image_outlier_rejected_count': 0, + 'image_outlier_sigma': COMPARISON_IMAGE_OUTLIER_SIGMA, + 'image_outlier_required_valid_pairs': 0, + 'image_outlier_available_pairs': 0, + 'image_outlier_valid_pair_counts': np.zeros(0, dtype=int), + 'image_outlier_outlier_pair_counts': np.zeros(0, dtype=int), + } - mask0 = image_data[0][roiy, roix] > np.percentile(image_data[0][roiy, roix], p) - mask0 = binary_erosion(mask0, iterations=it) + comp_keys = list(comp_flux_map.keys()) + normalized_flux_map = { + key: normalize_flux_series(comp_flux_map[key], validity_mask_func=validity_mask_func) + for key in comp_keys + } + coverage_summary = comparison_star_coverage_summary( + comp_flux_map, + skip_rejection=(skip_low_coverage_rejection or bypass_vetting), + validity_mask_func=validity_mask_func, + ) + coverage_qualified_keys = [ + key for key in comp_keys + if not coverage_summary[key]['coverage_rejected'] + ] + + def build_stability_iteration(active_keys, frame_keep_mask=None): + active_key_set = set(active_keys) + if comp_keys: + reference_shape = normalized_flux_map[comp_keys[0]].shape + else: + reference_shape = () + if frame_keep_mask is None: + working_flux_map = normalized_flux_map + else: + frame_keep_mask = np.asarray(frame_keep_mask, dtype=bool) + if frame_keep_mask.shape != reference_shape: + working_flux_map = normalized_flux_map + else: + working_flux_map = { + key: np.where(frame_keep_mask, normalized_flux_map[key], np.nan) + for key in comp_keys + } + active_flux_map = { + eligible_key: working_flux_map[eligible_key] + for eligible_key in active_keys + } + pairwise_matrix = np.full((len(comp_keys), len(comp_keys)), np.nan, dtype=float) + comp_summaries = [] - try: - results = aa.find_transform(mask1, mask0) - return results[0] - except Exception: - try: - result1 = ird.similarity(image_data[1][roiy, roix], image_data[0][roiy, roix], numiter=3) - return SimilarityTransform(scale=result1['scale'], rotation=np.radians(result1['angle']), - translation=[-1 * result1['tvec'][1], -1 * result1['tvec'][0]]) - except Exception: - pass + for i, key in enumerate(comp_keys): + normalized_flux = working_flux_map[key] + self_score = cheap_lightcurve_prescore(normalized_flux, np.ones(normalized_flux.shape[0]), airmass) + pairwise_scores = [] + pairwise_series = {} - log.debug(f"Warning: Following image failed to align - {file_name}") - plateStatus.alignmentError() - return SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) + for j, other_key in enumerate(comp_keys): + if i == j or other_key not in active_key_set: + continue + other_flux = working_flux_map[other_key] + score = cheap_lightcurve_prescore(normalized_flux, other_flux, airmass) + pairwise_matrix[i, j] = score + pairwise_series[f"vs {j + 1}"] = normalized_ratio_series(normalized_flux, other_flux) + if np.isfinite(score): + pairwise_scores.append(float(score)) + + ensemble_flux = build_normalized_comp_ensemble(active_flux_map, key) + ensemble_score = np.inf + ensemble_ratio_series = np.full(normalized_flux.shape, np.nan, dtype=float) + if ensemble_flux is not None: + ensemble_score = cheap_lightcurve_prescore(normalized_flux, ensemble_flux, airmass) + ensemble_ratio_series = normalized_ratio_series(normalized_flux, ensemble_flux) + + if pairwise_scores: + pairwise_median = float(np.nanmedian(pairwise_scores)) + pairwise_max = float(np.nanmax(pairwise_scores)) + pairwise_upper = float(np.nanpercentile(pairwise_scores, 75)) + else: + pairwise_median = np.inf + pairwise_max = np.inf + pairwise_upper = np.inf + + aggregate_inputs = [score for score in (ensemble_score, pairwise_upper) if np.isfinite(score)] + aggregate_score = max(aggregate_inputs) if aggregate_inputs else self_score + if coverage_summary[key]['coverage_rejected']: + aggregate_score = np.inf + + comp_summaries.append({ + 'comp_index': i, + 'key': key, + 'label': f"Comp {i + 1}", + 'pairwise_median_score': pairwise_median, + 'pairwise_max_score': pairwise_max, + 'ensemble_score': float(ensemble_score) if np.isfinite(ensemble_score) else np.inf, + 'self_score': float(self_score) if np.isfinite(self_score) else np.inf, + 'aggregate_score': float(aggregate_score) if np.isfinite(aggregate_score) else np.inf, + 'valid_pair_count': len(pairwise_scores), + 'pairwise_ratio_series': pairwise_series, + 'ensemble_ratio_series': ensemble_ratio_series, + 'coverage_count': coverage_summary[key]['coverage_count'], + 'coverage_total_frame_count': coverage_summary[key]['coverage_total_frame_count'], + 'coverage_reference_count': coverage_summary[key]['coverage_reference_count'], + 'coverage_min_required_count': coverage_summary[key]['coverage_min_required_count'], + 'coverage_rejected': coverage_summary[key]['coverage_rejected'], + 'coverage_rejection_threshold_count': coverage_summary[key].get('coverage_rejection_threshold_count'), + 'coverage_rejection_reference_count': coverage_summary[key].get('coverage_rejection_reference_count'), + 'coverage_rejection_scatter': coverage_summary[key].get('coverage_rejection_scatter'), + 'coverage_rejection_iteration': coverage_summary[key].get('coverage_rejection_iteration'), + 'suitability_outlier_rejected': False, + 'suitability_reference_score': np.nan, + 'suitability_scatter': np.nan, + 'suitability_high_threshold': np.nan, + }) + + return pairwise_matrix, comp_summaries + + active_keys = list(comp_keys if bypass_vetting else coverage_qualified_keys) + rejected_outlier_keys = set() + rejection_metadata = {} + for _ in range(0 if bypass_vetting else COMPARISON_STAR_SUITABILITY_MAX_ITERS): + _, iteration_summaries = build_stability_iteration(active_keys) + active_indices = [comp_keys.index(key) for key in active_keys] + outlier_summary = apply_comparison_star_suitability_outlier_rejection( + iteration_summaries, + eligible_indices=active_indices, + ) + newly_rejected_indices = outlier_summary['rejected_indices'] + if not newly_rejected_indices: + break + newly_rejected_keys = [comp_keys[index] for index in newly_rejected_indices] + if len(active_keys) - len(newly_rejected_keys) < 3: + break -def get_img_scale(hdr, wcs_file, pixel_init): - if wcs_file: - wcs_hdr = fits.getheader(wcs_file) - astrometry_scale = [key.value.split(' ') for key in wcs_hdr._cards if 'scale:' in str(key.value)] + for index in newly_rejected_indices: + key = comp_keys[index] + if key in rejection_metadata: + continue + rejection_metadata[key] = { + 'reference_score': outlier_summary['reference_score'], + 'scatter': outlier_summary['scatter'], + 'high_threshold': outlier_summary['high_threshold'], + } + rejected_outlier_keys.update(newly_rejected_keys) + active_keys = [key for key in active_keys if key not in rejected_outlier_keys] + + if bypass_vetting: + series_length = np.asarray(next(iter(normalized_flux_map.values()))).shape[0] + image_outlier_summary = { + 'frame_keep_mask': np.ones(series_length, dtype=bool), + 'rejected_frame_count': 0, + 'sigma': COMPARISON_IMAGE_OUTLIER_SIGMA, + 'required_valid_pair_count': 0, + 'available_pair_count': 0, + 'valid_pair_counts': np.zeros(series_length, dtype=int), + 'outlier_pair_counts': np.zeros(series_length, dtype=int), + } + else: + image_outlier_summary = comparison_star_image_outlier_summary( + normalized_flux_map, + active_keys, + ) + field_image_keep_mask = image_outlier_summary['frame_keep_mask'] + pairwise_matrix, comp_summaries = build_stability_iteration( + active_keys, + frame_keep_mask=field_image_keep_mask, + ) + final_active_scores = np.asarray( + [ + summary['aggregate_score'] + for summary in comp_summaries + if summary['key'] in set(active_keys) and np.isfinite(summary['aggregate_score']) + ], + dtype=float, + ) + final_reference_score = np.nan + final_scatter = np.nan + final_high_threshold = np.nan + if final_active_scores.size and np.any(np.isfinite(final_active_scores)): + final_reference_score = float(np.nanmedian(final_active_scores)) + final_scatter = robust_scatter(final_active_scores) + if np.isfinite(final_scatter) and final_scatter > 0: + final_high_threshold = ( + final_reference_score + COMPARISON_STAR_SUITABILITY_OUTLIER_SIGMA * final_scatter + ) - if astrometry_scale: - img_scale_num = astrometry_scale[0][1] - img_scale_units = astrometry_scale[0][2] + for summary in comp_summaries: + rejection_info = rejection_metadata.get(summary['key']) + if rejection_info is not None: + summary['suitability_outlier_rejected'] = True + summary['suitability_reference_score'] = rejection_info['reference_score'] + summary['suitability_scatter'] = rejection_info['scatter'] + summary['suitability_high_threshold'] = rejection_info['high_threshold'] else: - wcs = WCS(wcs_hdr).proj_plane_pixel_scales() - img_scale_num = (wcs[0].value + wcs[1].value) / 2 * 3600 # Convert to arcsec/pixel - img_scale_units = "arcsec/pixel" - elif 'IM_SCALE' in hdr: - img_scale_num = hdr['IM_SCALE'] - img_scale_units = hdr.comments['IM_SCALE'] - elif 'PIXSCALE' in hdr: - img_scale_num = hdr['PIXSCALE'] - img_scale_units = hdr.comments['PIXSCALE'] - elif pixel_init: - img_scale_num = pixel_init - img_scale_units = "arcsec/pixel" - else: - log_info("Not able to find Image Scale in the Image Header.") - img_scale_num = user_input("Please enter Image Scale (arcsec/pixel): ", type_=float) - img_scale_units = "arcsec/pixel" + summary['suitability_outlier_rejected'] = False + summary['suitability_reference_score'] = final_reference_score + summary['suitability_scatter'] = final_scatter + summary['suitability_high_threshold'] = final_high_threshold + + if bypass_vetting: + candidate_frame_summary = { + 'frame_keep_mask': np.ones(field_image_keep_mask.shape, dtype=bool), + 'rejected_frame_indices': [], + 'rejected_frame_count': 0, + 'valid_pair_counts': np.zeros(field_image_keep_mask.shape, dtype=int), + 'outlier_pair_counts': np.zeros(field_image_keep_mask.shape, dtype=int), + 'required_valid_pair_count': 0, + 'available_pair_count': 0, + 'sigma': COMPARISON_IMAGE_OUTLIER_SIGMA, + } + else: + candidate_frame_summary = comparison_star_candidate_frame_outlier_summary( + normalized_flux_map, + summary['key'], + active_keys, + field_image_keep_mask=field_image_keep_mask, + ) + summary['ensemble_frame_keep_mask'] = candidate_frame_summary['frame_keep_mask'] + summary['ensemble_frame_rejected_indices'] = candidate_frame_summary['rejected_frame_indices'] + summary['ensemble_frame_rejected_count'] = candidate_frame_summary['rejected_frame_count'] + summary['ensemble_frame_valid_pair_counts'] = candidate_frame_summary['valid_pair_counts'] + summary['ensemble_frame_outlier_pair_counts'] = candidate_frame_summary['outlier_pair_counts'] + summary['ensemble_frame_required_valid_pairs'] = candidate_frame_summary['required_valid_pair_count'] + summary['ensemble_frame_available_pairs'] = candidate_frame_summary['available_pair_count'] + summary['ensemble_frame_sigma'] = candidate_frame_summary['sigma'] + + finite_comp_scores = [ + summary['aggregate_score'] + for summary in comp_summaries + if ( + np.isfinite(summary['aggregate_score']) + and not summary.get('suitability_outlier_rejected') + ) + ] + field_score = float(np.nanmedian(finite_comp_scores)) if finite_comp_scores else np.inf + best_comp_index = None + best_comp_score = np.inf + for summary in comp_summaries: + if summary.get('suitability_outlier_rejected'): + continue + if summary['aggregate_score'] < best_comp_score: + best_comp_score = summary['aggregate_score'] + best_comp_index = summary['comp_index'] + + return { + 'pairwise_matrix': pairwise_matrix, + 'comp_summaries': comp_summaries, + 'field_score': field_score, + 'best_comp_index': best_comp_index, + 'best_comp_score': best_comp_score, + 'suitability_outlier_rejected_count': len(rejected_outlier_keys), + 'suitability_high_threshold': final_high_threshold, + 'suitability_reference_score': final_reference_score, + 'suitability_scatter': final_scatter, + 'field_image_keep_mask': field_image_keep_mask, + 'image_outlier_rejected_count': image_outlier_summary['rejected_frame_count'], + 'image_outlier_sigma': image_outlier_summary['sigma'], + 'image_outlier_required_valid_pairs': image_outlier_summary['required_valid_pair_count'], + 'image_outlier_available_pairs': image_outlier_summary['available_pair_count'], + 'image_outlier_valid_pair_counts': image_outlier_summary['valid_pair_counts'], + 'image_outlier_outlier_pair_counts': image_outlier_summary['outlier_pair_counts'], + } - img_scale = f"Image scale in {img_scale_units}: {round_to_2(float(img_scale_num))}" - return img_scale, float(img_scale_num) +def comparison_field_sort_value(value, zero_tolerance=1.0e-12): + if value is None: + return np.inf + try: + numeric_value = float(value) + except (TypeError, ValueError): + return np.inf + if not np.isfinite(numeric_value): + return np.inf + if abs(numeric_value) <= zero_tolerance: + return 0.0 + return numeric_value + + +def comparison_field_sort_key(summary): + return ( + comparison_field_sort_value(summary.get('field_score')), + comparison_field_sort_value(summary.get('best_comp_score')), + ) -def exp_time_med(exptimes): - # exposure time - consistent_et = False - if len(exptimes) > 0: - consistent_et = all(elem == exptimes[0] for elem in exptimes) +def initialize_aperture_data_store(frame_count, aperture_count, annulus_count, comp_star_count): + aper_shape = (frame_count, aperture_count, annulus_count) + aper_data = { + 'target': np.full(aper_shape, np.nan, dtype=float), + 'target_bg': np.full(aper_shape, np.nan, dtype=float), + 'target_unc': np.full(aper_shape, np.nan, dtype=float), + 'target_overexposed': np.zeros(frame_count, dtype=bool), + } + for component in NOISE_BUDGET_COMPONENT_KEYS: + aper_data[f"target_noise_{component}"] = np.full(aper_shape, np.nan, dtype=float) - exptimes = np.array(exptimes) + for comp_idx in range(comp_star_count): + ckey = f"comp{comp_idx + 1}" + aper_data[ckey] = np.full(aper_shape, np.nan, dtype=float) + aper_data[f"{ckey}_bg"] = np.full(aper_shape, np.nan, dtype=float) + aper_data[f"{ckey}_unc"] = np.full(aper_shape, np.nan, dtype=float) + aper_data[f"{ckey}_overexposed"] = np.zeros(frame_count, dtype=bool) + for component in NOISE_BUDGET_COMPONENT_KEYS: + aper_data[f"{ckey}_noise_{component}"] = np.full(aper_shape, np.nan, dtype=float) - if consistent_et: - return exptimes[0] - else: - return np.median(exptimes) + return aper_data -def update_coordinates_with_proper_motion(info_dict, time_obs): - parameter_names = { - 'dist': 'Distance (pc)', - 'pm_ra': 'Proper Motion RA (mas/yr)', - 'pm_dec': 'Proper Motion DEC (mas/yr)' - } +def aperture_estimation_comparison_stars(science_comp_stars, stellar_variability_only=False): + """Choose from comparison stars after the caller's VSX-variable rejection pass.""" + candidates = [list(position) for position in (science_comp_stars or [])] + if stellar_variability_only: + return candidates[:STELLAR_VARIABILITY_APERTURE_ESTIMATION_MAX_COMPARISONS] + return candidates - missing_values = [parameter_names[key] for key in ['dist', 'pm_ra', 'pm_dec'] if info_dict.get(key, 0.0) == 0.0] - if missing_values: - missing_values = ", ".join(missing_values) - log_info("Warning: Cannot account for proper motion due to missing values in: " - f"\n{missing_values}. Please re-run and fill in values in the initialization file to account " - f"for proper motion", warn=True) +def collapse_aperture_data_to_selected_grid_cell(aper_data, aperture_index, annulus_index): + if not isinstance(aper_data, dict): + return None - return info_dict['ra'], info_dict['dec'] - else: - time_j2000 = Time(2000.0, format='jyear') - time_obs = Time(time_obs, format='jd') + aperture_index = int(aperture_index) + annulus_index = int(annulus_index) + selected = {} + for key, values in aper_data.items(): + array = np.asarray(values) + if array.ndim == 3: + if not ( + 0 <= aperture_index < array.shape[1] + and 0 <= annulus_index < array.shape[2] + ): + raise IndexError( + f"Selected aperture grid cell [{aperture_index}, {annulus_index}] " + f"is outside {key} shape {array.shape}." + ) + selected[key] = np.array( + array[:, aperture_index:aperture_index + 1, annulus_index:annulus_index + 1], + copy=True, + ) + else: + selected[key] = np.array(array, copy=True) + return selected - coord = SkyCoord( - ra=info_dict['ra'] * u.deg, - dec=info_dict['dec'] * u.deg, - distance=info_dict['dist'] * u.pc, - pm_ra_cosdec=info_dict['pm_ra'] * u.mas / u.yr, - pm_dec=info_dict['pm_dec'] * u.mas / u.yr, - frame="icrs", - obstime=time_j2000 + +def aperture_frame_sigma_from_psf_data(psf_data, frame_index, fallback_sigma=np.nan, + comparison_indices=None): + if comparison_indices is None: + return psf_sigma_from_fit( + psf_data['target'][frame_index], + fallback_sigma=fallback_sigma, ) - updated_coord = coord.apply_space_motion(new_obstime=time_obs) - return updated_coord.ra.deg, updated_coord.dec.deg + comparison_sigmas = [] + for comp_idx in comparison_indices: + ckey = f"comp{int(comp_idx) + 1}" + if ckey not in psf_data: + continue + comp_sigma = psf_sigma_from_fit(psf_data[ckey][frame_index], fallback_sigma=np.nan) + if np.isfinite(comp_sigma) and comp_sigma > 0: + comparison_sigmas.append(float(comp_sigma)) + if comparison_sigmas: + return float(np.median(comparison_sigmas)) + return finite_positive_or_nan(fallback_sigma) + + +def mask_aperture_star_frame(aper_data, key, frame_index): + if not isinstance(aper_data, dict) or key not in aper_data: + return + frame_index = int(frame_index) + for suffix in ('', '_bg', '_unc'): + data_key = f"{key}{suffix}" + if data_key in aper_data: + aper_data[data_key][frame_index, :, :] = np.nan + for component in NOISE_BUDGET_COMPONENT_KEYS: + noise_key = f"{key}_noise_{component}" + if noise_key in aper_data: + aper_data[noise_key][frame_index, :, :] = np.nan + overexposed_key = f"{key}_overexposed" + if overexposed_key in aper_data: + aper_data[overexposed_key][frame_index] = True + + +def apply_overexposure_masks_to_aperture_frame(aper_data, frame_index, target_overexposed, + comp_overexposed_masks=None): + if target_overexposed: + mask_aperture_star_frame(aper_data, 'target', frame_index) + + if not isinstance(comp_overexposed_masks, dict): + return + for key, mask in comp_overexposed_masks.items(): + try: + is_overexposed = bool(np.asarray(mask, dtype=bool)[int(frame_index)]) + except (IndexError, TypeError, ValueError): + is_overexposed = False + if is_overexposed: + mask_aperture_star_frame(aper_data, key, frame_index) -def gaussian_psf(x, y, x0, y0, a, sigx, sigy, rot, b): - rx = (x - x0) * np.cos(rot) - (y - y0) * np.sin(rot) - ry = (x - x0) * np.sin(rot) + (y - y0) * np.cos(rot) - gausx = np.exp(-rx ** 2 / (2 * sigx ** 2)) - gausy = np.exp(-ry ** 2 / (2 * sigy ** 2)) - return a * gausx * gausy + b +def compute_star_aperture_grid(data, star_index, xc, yc, apertures, annuli, fast_mode=False, sigma_hint=np.nan, + aperture_correction_factors=None, noise_config=None, exposure_s=np.nan, + airmass=np.nan, return_noise=False): + apertures = np.asarray(apertures, dtype=float).reshape(-1) + annuli = np.asarray(annuli, dtype=float).reshape(-1) + flux_grid = np.full((len(apertures), len(annuli)), np.nan, dtype=float) + bg_grid = np.full((len(apertures), len(annuli)), np.nan, dtype=float) + noise_grids = empty_noise_budget_grids(flux_grid.shape) if return_noise else None + if not (np.isfinite(xc) and np.isfinite(yc)): + return (flux_grid, bg_grid, noise_grids) if return_noise else (flux_grid, bg_grid) -def mesh_box(pos, box, maxx=0, maxy=0): - pos = [int(np.round(pos[0])), int(np.round(pos[1]))] - if maxx: - x = np.arange(max(0,pos[0] - box), min(maxx, pos[0] + box + 1)) - else: - x = np.arange(max(0,pos[0] - box), pos[0] + box + 1) - if maxy: - y = np.arange(max(0,pos[1] - box), min(maxy, pos[1] + box + 1)) + mask_method = 'center' if fast_mode else 'exact' + + for a_idx, aperture_radius in enumerate(apertures): + if not np.isfinite(aperture_radius) or aperture_radius <= 0: + continue + aperture = CircularAperture(positions=[(xc, yc)], r=float(aperture_radius)) + mask = aperture.to_mask(method=mask_method)[0] + data_cutout = mask.cutout(data) + + mask_area = None + raw_aperture_sum = None + if data_cutout is not None: + mask_area = np.sum(mask.data) + raw_aperture_sum = (mask.data * data_cutout).sum() + + for an_idx, annulus_width in enumerate(annuli): + if not np.isfinite(annulus_width) or annulus_width < 0: + continue + stage_start = perf_counter() + try: + if annulus_width > 0: + sky_geometry = resolve_sky_annulus_geometry( + aperture_radius=float(aperture_radius), + annulus_width=float(annulus_width), + psf_sigma=sigma_hint, + ) + bgflux, sigmabg, n_sky = skybg_phot( + data, + star_index, + xc, + yc, + sky_geometry['inner_radius'], + sky_geometry['annulus_width'], + fast_mode=fast_mode, + ) + else: + bgflux = 0 + sigmabg = 0 + n_sky = 0 + + bg_grid[a_idx, an_idx] = bgflux + + if data_cutout is None: + flux_grid[a_idx, an_idx] = 0 + else: + flux_grid[a_idx, an_idx] = raw_aperture_sum - bgflux * mask_area + if return_noise and noise_grids is not None: + budget = compute_photometry_noise_budget( + flux_grid[a_idx, an_idx], + sigmabg, + mask_area, + n_sky, + exposure_s=exposure_s, + airmass=airmass, + noise_config=noise_config, + ) + for component, grid in noise_grids.items(): + grid[a_idx, an_idx] = budget.get(component, np.nan) + finally: + _record_photometry_stage_timing('aperPhot', perf_counter() - stage_start) + + if aperture_correction_factors is not None: + factors = np.asarray(aperture_correction_factors, dtype=float).reshape(-1) + if factors.shape[0] == len(apertures): + valid_factors = np.isfinite(factors) & (factors > 0) + if np.any(valid_factors): + flux_grid[valid_factors, :] *= factors[valid_factors, None] + if return_noise and noise_grids is not None: + for grid in noise_grids.values(): + grid[valid_factors, :] *= factors[valid_factors, None] + + return (flux_grid, bg_grid, noise_grids) if return_noise else (flux_grid, bg_grid) + + +def populate_aperture_data_for_frame(image_data, frame_index, psf_data, comp_star_count, aper_data, apertures, annuli, + fast_aperture_mask, adaptive_apertures=False, fallback_sigma=np.nan, + use_aperture_corrections_and_full_image_fwhm=False, + noise_config=None, exposure_s=np.nan, airmass=np.nan, + comp_indices=None, include_target=True, + frame_sigma_comp_indices=None): + frame_seed_sigma = aperture_frame_sigma_from_psf_data( + psf_data, + frame_index, + fallback_sigma=fallback_sigma, + comparison_indices=frame_sigma_comp_indices, + ) + frame_seed_fwhm = psf_fwhm_from_sigma(frame_seed_sigma) + field_star_psfs = np.empty((0, 7), dtype=float) + image_fwhm = frame_seed_fwhm + if use_aperture_corrections_and_full_image_fwhm: + field_star_psfs = estimate_isolated_field_star_psfs(image_data, fwhm_hint=frame_seed_fwhm) + image_fwhm = image_fwhm_from_field_star_psfs(field_star_psfs, fallback_fwhm=frame_seed_fwhm) + frame_sigma = ( + image_fwhm / GAUSSIAN_SIGMA_TO_FWHM + if np.isfinite(image_fwhm) and image_fwhm > 0 + else frame_seed_sigma + ) + frame_apertures, frame_annuli = resolve_frame_aperture_radii( + apertures, + annuli, + adaptive_apertures=adaptive_apertures, + frame_sigma=frame_sigma, + fallback_sigma=fallback_sigma, + ) + aperture_correction = { + 'applied': False, + 'image_fwhm': image_fwhm, + 'star_count': 0, + 'correction_factors': np.ones(len(frame_apertures), dtype=float), + 'note': 'Aperture corrections and full-image FWHM estimation disabled.', + } + aperture_correction_factors = None + if use_aperture_corrections_and_full_image_fwhm: + aperture_correction = build_aperture_correction_profile( + image_data, + frame_apertures, + fwhm_hint=image_fwhm, + fast_mode=fast_aperture_mask, + field_star_psfs=field_star_psfs, + ) + aperture_correction_factors = aperture_correction.get('correction_factors') + + if include_target: + target_flux, target_bg, target_noise = compute_star_aperture_grid( + image_data, + 0, + psf_data['target'][frame_index, 0], + psf_data['target'][frame_index, 1], + frame_apertures, + frame_annuli, + fast_mode=fast_aperture_mask, + sigma_hint=frame_sigma, + aperture_correction_factors=aperture_correction_factors, + noise_config=noise_config, + exposure_s=exposure_s, + airmass=airmass, + return_noise=True, + ) + aper_data['target'][frame_index] = target_flux + aper_data['target_bg'][frame_index] = target_bg + aper_data['target_unc'][frame_index] = target_noise['total'] + for component in NOISE_BUDGET_COMPONENT_KEYS: + aper_data[f"target_noise_{component}"][frame_index] = target_noise[component] + + if comp_indices is None: + selected_comp_indices = range(comp_star_count) else: - y = np.arange(max(0,pos[1] - box), pos[1] + box + 1) - xv, yv = np.meshgrid(x, y) - return xv.astype(int), yv.astype(int) + selected_comp_indices = sorted({ + int(comp_idx) + for comp_idx in comp_indices + if 0 <= int(comp_idx) < int(comp_star_count) + }) + + for comp_idx in selected_comp_indices: + ckey = f"comp{comp_idx + 1}" + comp_sigma = psf_sigma_from_fit(psf_data[ckey][frame_index], fallback_sigma=frame_sigma) + comp_flux, comp_bg, comp_noise = compute_star_aperture_grid( + image_data, + comp_idx + 1, + psf_data[ckey][frame_index, 0], + psf_data[ckey][frame_index, 1], + frame_apertures, + frame_annuli, + fast_mode=fast_aperture_mask, + sigma_hint=comp_sigma, + aperture_correction_factors=aperture_correction_factors, + noise_config=noise_config, + exposure_s=exposure_s, + airmass=airmass, + return_noise=True, + ) + aper_data[ckey][frame_index] = comp_flux + aper_data[f"{ckey}_bg"][frame_index] = comp_bg + aper_data[f"{ckey}_unc"][frame_index] = comp_noise['total'] + for component in NOISE_BUDGET_COMPONENT_KEYS: + aper_data[f"{ckey}_noise_{component}"][frame_index] = comp_noise[component] + + return aperture_correction + + +def load_calibrated_reduction_image(file_name, generalDark, generalBias, generalFlat, + demosaic_fmt, demosaic_out, demosaic_mult, + bad_pixel_reference=None): + _, image_data = load_calibrated_reduction_frame( + file_name, + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + ) + return image_data -# Method fits a 2D gaussian function that matches the star_psf to the star image and returns its pixel coordinates -def fit_centroid(data, pos, starIndex, psf_function=gaussian_psf, box=15, weightedcenter=True): - # get sub field in image - xv, yv = mesh_box(pos, box, maxx=data.shape[1], maxy=data.shape[0]) - subarray = data[yv, xv] +def load_calibrated_reduction_frame(file_name, generalDark, generalBias, generalFlat, + demosaic_fmt, demosaic_out, demosaic_mult, + bad_pixel_reference=None): + hdul = fits.open(name=file_name, memmap=False, cache=False, lazy_load_hdus=False, ignore_missing_end=True) + extension = 0 + image_header = hdul[extension].header + while image_header["NAXIS"] == 0: + extension += 1 + image_header = hdul[extension].header + + image_data = hdul[extension].data + hdul.close() + + image_data = apply_cals(image_data, generalDark, generalBias, generalFlat, 1) + image_data = demosaic_img(image_data, demosaic_fmt, demosaic_out, demosaic_mult, 1) + image_data = repair_bad_pixels_in_frame(image_data, bad_pixel_reference) + return image_header, image_data + + +def ensure_first_reduction_image_for_fov(first_image, first_file_name, + generalDark, generalBias, generalFlat, + demosaic_fmt, demosaic_out, demosaic_mult, + bad_pixel_reference=None): + """Lazily load the first calibrated frame when multiprocessing did not retain it.""" + if first_image is not None: + return first_image + return load_calibrated_reduction_image( + first_file_name, + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + ) + + +def evenly_spaced_aperture_tuning_indices(frame_count, max_frames=APERTURE_AUTOTUNE_MAX_FRAMES, + min_frames=APERTURE_AUTOTUNE_MIN_FRAMES): + """Select representative frames from the beginning through the end of a run.""" + frame_count = max(0, int(frame_count)) + if frame_count == 0: + return np.array([], dtype=int) + + requested = min(frame_count, max(1, int(max_frames))) + if frame_count >= int(min_frames): + requested = max(int(min_frames), requested) + return np.linspace(0, frame_count - 1, requested, dtype=int) + + +def centered_numpy_cutout(data, xc, yc, radius): + """Copy only the square slice needed for local aperture measurements.""" + data = np.asarray(data) + if data.ndim != 2 or not (np.isfinite(xc) and np.isfinite(yc) and np.isfinite(radius)): + return None, np.nan, np.nan + + radius = max(float(radius), 1.0) + x0 = max(0, int(np.floor(float(xc) - radius))) + x1 = min(data.shape[1], int(np.ceil(float(xc) + radius)) + 1) + y0 = max(0, int(np.floor(float(yc) - radius))) + y1 = min(data.shape[0], int(np.ceil(float(yc) + radius)) + 1) + if x1 <= x0 or y1 <= y0: + return None, np.nan, np.nan + return np.array(data[y0:y1, x0:x1], copy=True), float(xc) - x0, float(yc) - y0 + + +def can_memmap_aperture_tuning_cutouts(generalDark=None, generalBias=None, generalFlat=None, + demosaic_fmt=None, bad_pixel_reference=None): + calibration_arrays = (generalDark, generalBias, generalFlat) + has_calibration = any( + value is not None and np.asarray(value).size > 0 + for value in calibration_arrays + ) + return bool( + not has_calibration + and not demosaic_fmt + and bad_pixel_reference is None + ) + + +def fits_header_supports_memmap(header): + """Astropy cannot expose scaled FITS image arrays through a memory map.""" try: - init = [np.nanmax(subarray) - np.nanmin(subarray), 1, 1, 0, np.nanmin(subarray)] - except ValueError as ve: - # Handle null subfield - cannot solve - plateStatus.outOfFrameWarning(starIndex) - log.debug(f"Warning: empty subfield for fit_centroid at {np.round(pos, 2)}") - return np.empty(7) * np.nan - # compute flux weighted centroid in x and y - wx = np.sum(xv[0]*subarray.sum(0))/subarray.sum(0).sum() - wy = np.sum(yv[:,0]*subarray.sum(1))/subarray.sum(1).sum() + bscale = float(header.get('BSCALE', 1.0)) + bzero = float(header.get('BZERO', 0.0)) + except (TypeError, ValueError): + return False + return bool(bscale == 1.0 and bzero == 0.0) + + +def _open_memmapped_reduction_frame(file_name): + hdul = fits.open( + name=file_name, + memmap=True, + cache=False, + lazy_load_hdus=True, + ignore_missing_end=True, + ) + extension = 0 + image_header = hdul[extension].header + while image_header["NAXIS"] == 0: + extension += 1 + image_header = hdul[extension].header + if not fits_header_supports_memmap(image_header): + hdul.close() + raise ValueError("Scaled FITS image requires the non-memmap reduction path.") + return hdul, image_header, hdul[extension].data + + +def build_aperture_tuning_cutouts(inputfiles, frame_indices, psf_data, comparison_indices, + adaptive_apertures, reference_sigma, + generalDark=None, generalBias=None, generalFlat=None, + demosaic_fmt=None, demosaic_out=None, demosaic_mult=None, + bad_pixel_reference=None, p_dict=None, info_dict=None, + jd_times=None, reject_overexposed=False, + overexposure_threshold=np.nan, fast_aperture_mask=False): + """Read each tuning frame once and retain compact star-local NumPy slices.""" + comparison_indices = tuple(int(index) for index in comparison_indices) + frame_indices = np.asarray(frame_indices, dtype=int) + frames = [] + sample_airmass = [] + sample_overexposed_masks = { + f"comp{comp_idx + 1}": np.zeros(len(frame_indices), dtype=bool) + for comp_idx in comparison_indices + } + use_memmap = can_memmap_aperture_tuning_cutouts( + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + bad_pixel_reference=bad_pixel_reference, + ) - # lower bound: [xc, yc, amp, sigx, sigy, rotation, bg] - lo = [pos[0] - box * 0.5, pos[1] - box * 0.5, 0, 0.5, 0.5, -np.pi / 4, np.nanmin(subarray) - 1] - up = [pos[0] + box * 0.5, pos[1] + box * 0.5, 1e7, 20, 20, np.pi / 4, np.nanmax(subarray) + 1] + for sample_index, frame_index in enumerate(frame_indices): + memmap_hdul = None + if use_memmap: + try: + memmap_hdul, header, image_data = _open_memmapped_reduction_frame( + inputfiles[frame_index] + ) + except (OSError, ValueError, TypeError): + if memmap_hdul is not None: + memmap_hdul.close() + memmap_hdul = None + header, image_data = load_calibrated_reduction_frame( + inputfiles[frame_index], + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + ) + else: + header, image_data = load_calibrated_reduction_frame( + inputfiles[frame_index], + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + ) + frame_sigma = aperture_frame_sigma_from_psf_data( + psf_data, + frame_index, + fallback_sigma=reference_sigma, + comparison_indices=comparison_indices, + ) + if not np.isfinite(frame_sigma) or frame_sigma <= 0: + frame_sigma = finite_positive_or_nan(reference_sigma) + if not np.isfinite(frame_sigma) or frame_sigma <= 0: + frame_sigma = 1.0 + + max_aperture, max_annulus = resolve_frame_aperture_radii( + [APERTURE_SIGMA_MAX if adaptive_apertures else APERTURE_SIGMA_MAX * reference_sigma], + [ANNULUS_SIGMA_MAX if adaptive_apertures else ANNULUS_SIGMA_MAX * reference_sigma], + adaptive_apertures=adaptive_apertures, + frame_sigma=frame_sigma, + fallback_sigma=reference_sigma, + ) + max_geometry = resolve_sky_annulus_geometry( + float(max_aperture[0]), + float(max_annulus[0]), + psf_sigma=frame_sigma, + ) + cutout_radius = float( + max_geometry['inner_radius'] + max_geometry['annulus_width'] + 2.0 + ) - def fcn2min(pars): - model = psf_function(xv, yv, *pars) - return (subarray - model).flatten() + stars = {} + for comp_idx in comparison_indices: + ckey = f"comp{comp_idx + 1}" + row = np.asarray(psf_data[ckey][frame_index], dtype=float) + xc = row[0] if row.size > 0 else np.nan + yc = row[1] if row.size > 1 else np.nan + cutout, local_xc, local_yc = centered_numpy_cutout( + image_data, + xc, + yc, + cutout_radius, + ) + stars[ckey] = { + 'data': cutout, + 'xc': local_xc, + 'yc': local_yc, + 'sigma': psf_sigma_from_fit(row, fallback_sigma=frame_sigma), + } + if reject_overexposed: + sample_overexposed_masks[ckey][sample_index] = aperture_contains_overexposed_pixel( + image_data, + xc, + yc, + overexposure_aperture_radius_from_psf_row(row, fallback_sigma=frame_sigma), + overexposure_threshold, + fast_mode=fast_aperture_mask, + ) + + if p_dict is not None and info_dict is not None and jd_times is not None: + sample_airmass.append(air_mass( + header, + p_dict['ra'], + p_dict['dec'], + info_dict['lat'], + info_dict['long'], + info_dict['elev'], + jd_times[frame_index], + )) + else: + sample_airmass.append(np.nan) + frames.append({'frame_sigma': frame_sigma, 'stars': stars}) + del image_data + if memmap_hdul is not None: + memmap_hdul.close() + + return frames, np.asarray(sample_airmass, dtype=float), sample_overexposed_masks + + +def populate_aperture_tuning_data_from_cutouts(cutout_frames, apertures, annuli, + comparison_indices, adaptive_apertures, + reference_sigma, fast_aperture_mask=False): + """Evaluate an aperture grid using only compact per-star image slices.""" + comparison_indices = tuple(int(index) for index in comparison_indices) + comp_star_count = max(comparison_indices, default=-1) + 1 + aper_data = initialize_aperture_data_store( + len(cutout_frames), + len(apertures), + len(annuli), + comp_star_count, + ) + for frame_index, frame in enumerate(cutout_frames): + frame_sigma = finite_positive_or_nan(frame.get('frame_sigma')) + if not np.isfinite(frame_sigma) or frame_sigma <= 0: + frame_sigma = finite_positive_or_nan(reference_sigma) + if not np.isfinite(frame_sigma) or frame_sigma <= 0: + frame_sigma = 1.0 + frame_apertures, frame_annuli = resolve_frame_aperture_radii( + apertures, + annuli, + adaptive_apertures=adaptive_apertures, + frame_sigma=frame_sigma, + fallback_sigma=reference_sigma, + ) + for comp_idx in comparison_indices: + ckey = f"comp{comp_idx + 1}" + star = frame['stars'].get(ckey, {}) + cutout = star.get('data') + if cutout is None: + continue + flux, background = compute_star_aperture_grid( + cutout, + comp_idx + 1, + star.get('xc', np.nan), + star.get('yc', np.nan), + frame_apertures, + frame_annuli, + fast_mode=fast_aperture_mask, + sigma_hint=star.get('sigma', frame_sigma), + return_noise=False, + ) + aper_data[ckey][frame_index] = flux + aper_data[f"{ckey}_bg"][frame_index] = background + return aper_data + + +def _refined_sigma_grid(center, lower_bound, upper_bound, half_width, points): + low = max(lower_bound, center - half_width) + high = min(upper_bound, center + half_width) + if high <= low: + low, high = lower_bound, upper_bound + return np.linspace(low, high, points) + + +def auto_tune_aperture_sigma_grid(coarse_apertures_sigma, coarse_annuli_sigma, coarse_aper_data, comp_star_count, + subset_airmass, require_comp_star=True, + skip_low_comparison_coverage_rejection=False, + psf_quality_masks=None): + best_candidate = None + best_score = np.inf + best_sort_key = (np.inf, np.inf) + + for a_idx, aperture_sigma in enumerate(coarse_apertures_sigma): + for an_idx, annulus_sigma in enumerate(coarse_annuli_sigma): + comp_flux_map = { + f"comp{comp_idx + 1}": mask_series_with_quality( + coarse_aper_data[f"comp{comp_idx + 1}"][:, a_idx, an_idx], + (psf_quality_masks or {}).get( + f"comp{comp_idx + 1}", + np.ones(coarse_aper_data[f"comp{comp_idx + 1}"].shape[0], dtype=bool), + ), + ) + for comp_idx in range(comp_star_count) + } + field_summary = comparison_star_stability_summary( + comp_flux_map, + subset_airmass, + skip_low_coverage_rejection=skip_low_comparison_coverage_rejection, + ) + field_score = field_summary['field_score'] + candidate_sort_key = comparison_field_sort_key(field_summary) + if np.isfinite(field_score) and candidate_sort_key < best_sort_key: + best_score = field_score + best_sort_key = candidate_sort_key + best_candidate = { + 'aper_sigma': float(aperture_sigma), + 'annulus_sigma': float(annulus_sigma), + 'comp_index': field_summary['best_comp_index'], + } + + if best_candidate is None: + center_aper_sigma = float(np.median(coarse_apertures_sigma)) + center_annulus_sigma = float(np.median(coarse_annuli_sigma)) + fallback_comp_index = None + if comp_star_count > 0: + fallback_comp_index = 0 + elif require_comp_star: + fallback_comp_index = None + best_candidate = { + 'aper_sigma': center_aper_sigma, + 'annulus_sigma': center_annulus_sigma, + 'comp_index': fallback_comp_index, + } + else: + center_aper_sigma = best_candidate['aper_sigma'] + center_annulus_sigma = best_candidate['annulus_sigma'] + + refined_apertures_sigma = _refined_sigma_grid( + center_aper_sigma, + APERTURE_SIGMA_MIN, + APERTURE_SIGMA_MAX, + APERTURE_AUTOTUNE_APER_HALF_WIDTH_SIGMA, + APERTURE_AUTOTUNE_REFINED_APER_POINTS, + ) + refined_annuli_sigma = _refined_sigma_grid( + center_annulus_sigma, + ANNULUS_SIGMA_MIN, + ANNULUS_SIGMA_MAX, + APERTURE_AUTOTUNE_ANNULUS_HALF_WIDTH_SIGMA, + APERTURE_AUTOTUNE_REFINED_ANNULUS_POINTS, + ) - try: - res = least_squares(fcn2min, x0=[*pos, *init], bounds=[lo, up], jac='3-point', xtol=None, method='trf') - except: - # Report low flux warning - plateStatus.lowFluxAmplitudeWarning(starIndex, pos[0], pos[1]) - log.debug(f"Warning: Measured flux amplitude is really low---are you sure there is a star at {np.round(pos, 2)}?") + return refined_apertures_sigma, refined_annuli_sigma, best_candidate, best_score - res = least_squares(fcn2min, x0=[*pos, *init], jac='3-point', xtol=None, method='lm') - # override psf fit results with weighted centroid - if weightedcenter: - res.x[0] = wx - res.x[1] = wy +def comparison_method_label(candidate): + if candidate.get('method') == 'psf': + return "PSF photometry" + return f"Aperture photometry (aper={candidate['aper']:.2f}px, annulus={candidate['annulus']:.2f}px)" - return res.x +def select_comparison_calibrated_photometry(psf_data, aper_data, apers, annuli, airmass, comp_stars, sigma, + skip_low_comparison_coverage_rejection=False, + use_psf_photometry=True, + use_aperture_photometry=True, + psf_flux_data=None, + comp_overexposed_masks=None, + use_exactly_the_comps_provided=False): + candidate_summaries = [] + comp_star_count = len(comp_stars) + psf_flux_data = psf_flux_data_source(psf_data, psf_flux_data) -# Method calculates the flux of the star (uses the skybg_phot method to do background sub) -def aperPhot(data, starIndex, xc, yc, r=5, dr=5): - # Check for invalid coordinates - if np.isnan(xc) or np.isnan(yc): - return 0, 0 - - # Calculate background if dr > 0 - if dr > 0: - bgflux, sigmabg, Nbg = skybg_phot(data, starIndex, xc, yc, r + 2, dr) - else: - bgflux, sigmabg, Nbg = 0, 0, 0 - - # Create aperture and mask - aperture = CircularAperture(positions=[(xc, yc)], r=r) - mask = aperture.to_mask(method='exact')[0] - data_cutout = mask.cutout(data) - - # Check if aperture is valid - if data_cutout is None: - # Aperture is partially or fully outside the image - return 0, bgflux # Return zero flux but valid background - - # Calculate and return aperture sum - aperture_sum = (mask.data * (data_cutout - bgflux)).sum() - return aperture_sum, bgflux - - -def skybg_phot(data, starIndex, xc, yc, r=10, dr=5, ptol=99, debug=False): - # create a crude annulus to mask out bright background pixels - # the box will not extend beyond the borders of the image - image_height, image_width = data.shape - xv, yv = mesh_box([xc, yc], np.round(r + dr), maxx=image_width, maxy=image_height) - rv = ((xv - xc) ** 2 + (yv - yc) ** 2) ** 0.5 - mask = (rv > r) & (rv < (r + dr)) - try: - cutoff = np.nanpercentile(data[yv, xv][mask], ptol) - except IndexError: - plateStatus.skyBackgroundWarning(starIndex, xc, yc) - log.debug(f"Warning: IndexError, problem computing sky bg for {xc:.1f}, {yc:.1f}." - f"\nCheck if star is present or close to border.") + if comp_star_count == 0: + return None - # create pixel wise mask on entire image - x = np.arange(data.shape[1]) - y = np.arange(data.shape[0]) - xv, yv = np.meshgrid(x, y) - rv = ((xv - xc) ** 2 + (yv - yc) ** 2) ** 0.5 - mask = (rv > r) & (rv < (r + dr)) - cutoff = np.nanpercentile(data[yv, xv][mask], ptol) + frame_count = len(airmass) + overexposure_masks = {} + for comp_idx in range(comp_star_count): + ckey = f"comp{comp_idx + 1}" + mask = None + if isinstance(comp_overexposed_masks, dict) and ckey in comp_overexposed_masks: + mask = np.asarray(comp_overexposed_masks[ckey], dtype=bool) + elif isinstance(aper_data, dict) and f"{ckey}_overexposed" in aper_data: + mask = np.asarray(aper_data[f"{ckey}_overexposed"], dtype=bool) + if mask is not None and mask.shape[0] == frame_count: + overexposure_masks[ckey] = mask + else: + overexposure_masks[ckey] = np.zeros(frame_count, dtype=bool) + centroid_psf_quality_masks = { + f"comp{comp_idx + 1}": ( + psf_quality_mask_for_key(psf_data, f"comp{comp_idx + 1}", frame_count) + & ~overexposure_masks[f"comp{comp_idx + 1}"] + ) + for comp_idx in range(comp_star_count) + } + psf_quality_masks = { + f"comp{comp_idx + 1}": ( + psf_quality_mask_for_key( + psf_data, + f"comp{comp_idx + 1}", + frame_count, + psf_flux_data=psf_flux_data, + ) + & ~overexposure_masks[f"comp{comp_idx + 1}"] + ) + for comp_idx in range(comp_star_count) + } - dat = np.array(data[yv, xv], dtype=float) - dat[dat > cutoff] = np.nan # ignore pixels brighter than percentile + if use_psf_photometry: + psf_flux_map = { + f"comp{comp_idx + 1}": psf_flux_series_from_rows( + psf_flux_data[f"comp{comp_idx + 1}"], + psf_quality_masks[f"comp{comp_idx + 1}"], + ) + for comp_idx in range(comp_star_count) + } + psf_summary = comparison_star_stability_summary( + psf_flux_map, + airmass, + skip_low_coverage_rejection=skip_low_comparison_coverage_rejection, + validity_mask_func=robust_flux_floor_mask, + bypass_vetting=use_exactly_the_comps_provided, + ) + psf_summary.update({ + 'method': 'psf', + 'a': None, + 'an': None, + 'aper': 0.0, + 'annulus': float(15 * sigma), + }) + candidate_summaries.append(psf_summary) + + if use_aperture_photometry and aper_data is not None and apers is not None and annuli is not None: + for a_idx, aperture in enumerate(apers): + for an_idx, annulus in enumerate(annuli): + comp_flux_map = { + f"comp{comp_idx + 1}": mask_series_with_quality( + aper_data[f"comp{comp_idx + 1}"][:, a_idx, an_idx], + centroid_psf_quality_masks[f"comp{comp_idx + 1}"], + ) + for comp_idx in range(comp_star_count) + } + candidate_summary = comparison_star_stability_summary( + comp_flux_map, + airmass, + skip_low_coverage_rejection=skip_low_comparison_coverage_rejection, + bypass_vetting=use_exactly_the_comps_provided, + ) + candidate_summary.update({ + 'method': 'aperture', + 'a': a_idx, + 'an': an_idx, + 'aper': float(aperture), + 'annulus': float(annulus), + }) + candidate_summaries.append(candidate_summary) + + finite_candidates = [ + candidate for candidate in candidate_summaries + if np.isfinite(candidate['field_score']) and candidate['best_comp_index'] is not None + ] + if finite_candidates: + finite_candidates.sort(key=comparison_field_sort_key) + best_candidate = finite_candidates[0] + elif use_exactly_the_comps_provided and candidate_summaries: + # Stability is diagnostic-only in exact mode. Keep a photometry + # method available even when the supplied stars have no finite field + # score; the later light-curve construction will either measure every + # requested star or fail explicitly without substitution. + best_candidate = candidate_summaries[0] + if best_candidate.get('best_comp_index') is None: + best_candidate['best_comp_index'] = 0 + best_candidate['best_comp_score'] = np.inf + else: + return None - if debug: - minb = data[yv, xv][mask].min() - maxb = data[yv, xv][mask].mean() + 3 * data[yv, xv][mask].std() - nanmask = np.nan * np.zeros(mask.shape) - nanmask[mask] = 1 - bgsky = data[yv, xv] * nanmask - cmode = mode(dat.flatten(), nan_policy='omit', keepdims=True).mode[0] - amode = mode(bgsky.flatten(), nan_policy='omit', keepdims=True).mode[0] + best_comp_index = best_candidate['best_comp_index'] + method_label = comparison_method_label(best_candidate) + comp_summaries = [] + best_quality_masks = psf_quality_masks if best_candidate.get('method') == 'psf' else centroid_psf_quality_masks + for summary in best_candidate['comp_summaries']: + comp_summary = dict(summary) + comp_summary['position'] = comp_stars[comp_summary['comp_index']] + comp_summary['selected'] = comp_summary['comp_index'] == best_comp_index + quality_mask = best_quality_masks.get(comp_summary['key']) + if quality_mask is not None: + comp_summary['psf_quality_keep_mask'] = quality_mask + comp_summary['psf_quality_rejected_count'] = int(np.count_nonzero(~quality_mask)) + else: + comp_summary['psf_quality_rejected_count'] = 0 + overexposure_mask = overexposure_masks.get(comp_summary['key']) + comp_summary['overexposure_rejected_count'] = ( + int(np.count_nonzero(overexposure_mask)) + if overexposure_mask is not None else 0 + ) + comp_summary['selection_reason'] = comparison_calibration_selection_reason( + comp_summary, + best_candidate['best_comp_score'], + ) + comp_summaries.append(comp_summary) - fig, ax = plt.subplots(2, 2, figsize=(9, 9)) - im = ax[0, 0].imshow(data[yv, xv], vmin=minb, vmax=maxb, cmap='inferno') - ax[0, 0].set_title("Original Data") - from mpl_toolkits.axes_grid1 import make_axes_locatable - divider = make_axes_locatable(ax[0, 0]) - cax = divider.append_axes('right', size='5%', pad=0.05) - fig.colorbar(im, cax=cax, orientation='vertical') + best_candidate['comp_summaries'] = comp_summaries + best_candidate['method_label'] = method_label + return best_candidate - ax[1, 0].hist(bgsky.flatten(), label=f'Sky Annulus ({np.nanmedian(bgsky):.1f}, {amode:.1f})', - alpha=0.5, bins=np.arange(minb, maxb)) - ax[1, 0].hist(dat.flatten(), label=f'Clipped ({np.nanmedian(dat):.1f}, {cmode:.1f})', alpha=0.5, - bins=np.arange(minb, maxb)) - ax[1, 0].legend(loc='best') - ax[1, 0].set_title("Sky Background") - ax[1, 0].set_xlabel("Pixel Value") - ax[1, 1].imshow(dat, vmin=minb, vmax=maxb, cmap='inferno') - ax[1, 1].set_title("Clipped Sky Background") +def ranked_comparison_calibration_summaries(comparison_calibration, include_unvetted=False): + if comparison_calibration is None: + return [] - ax[0, 1].imshow(bgsky, vmin=minb, vmax=maxb, cmap='inferno') - ax[0, 1].set_title("Sky Annulus") - plt.tight_layout() - plt.show() - return mode(dat.flatten(), nan_policy='omit', keepdims=True).mode[0], np.nanstd(dat.flatten()), np.sum(mask) + ranked_summaries = [] + for summary in comparison_calibration.get('comp_summaries', []): + aggregate_score = summary.get('aggregate_score', np.inf) + if not include_unvetted and summary.get('coverage_rejected'): + continue + if not include_unvetted and summary.get('suitability_outlier_rejected'): + continue + if not include_unvetted and not np.isfinite(aggregate_score): + continue + ranked_summaries.append(summary) -def process_dark_frames(dark_files): - """Process dark frames and return the master dark.""" - if not dark_files: - return None - # Dark files whose median is much higher than the overall dark files median will be filtered - # e.g. to discard saturated dark files that may negatively affect the master dark used to calibrate the science frames - # First pass: collect all dark frame medians - darks_medians = [(dark_file, np.nanmedian(fits.getdata(dark_file))) for dark_file in dark_files] + if include_unvetted: + ranked_summaries.sort(key=lambda summary: summary.get('comp_index', np.inf)) + else: + ranked_summaries.sort( + key=lambda summary: ( + summary.get('aggregate_score', np.inf), + summary.get('comp_index', np.inf), + ) + ) + return ranked_summaries - d_median = np.median([median for _, median in darks_medians]) - threshold = 1.7 # 70% higher than overall median - # Second pass: collect valid dark frames - darks_img_list = [] - for dark_file, dark_median in darks_medians: - median_ratio = dark_median / d_median - if median_ratio > threshold: - log_info( - f"\nWarning: Skipping suspicious dark frame {dark_file}: " - f"median/overall_median = {median_ratio:.2f}\n", - warn=True - ) - continue - dark_data = fits.getdata(dark_file) - darks_img_list.append(dark_data) - - return np.median(darks_img_list, axis=0) if darks_img_list else None +def comparison_positions_match(first, second, tolerance_pixels=1.0e-6): + try: + first_values = np.asarray(first, dtype=float).reshape(-1) + second_values = np.asarray(second, dtype=float).reshape(-1) + except (TypeError, ValueError): + return False + return ( + first_values.size >= 2 + and second_values.size >= 2 + and np.all(np.isfinite(first_values[:2])) + and np.all(np.isfinite(second_values[:2])) + and np.allclose(first_values[:2], second_values[:2], rtol=0.0, atol=float(tolerance_pixels)) + ) -def process_bias_frames(bias_files): - """Process bias frames and return the master bias.""" - if not bias_files: - return None - - biases_img_list = [fits.getdata(bias_file) for bias_file in bias_files] - return np.median(biases_img_list, axis=0) if biases_img_list else None -def process_flat_frames(flat_files, master_bias=None): - """Process flat frames and return the normalized master flat.""" - if not flat_files: +def stellar_variability_calibration_for_position(calibration_stars, position, observed_filter=None): + candidates = [] + for label, star in (calibration_stars or {}).items(): + if not isinstance(star, dict) or not comparison_positions_match(star.get('pos'), position): + continue + magnitude = _finite_float(star.get('mag')) + magnitude_error = normalized_magnitude_error(star.get('error')) + if not is_usable_apparent_magnitude(magnitude) or magnitude_error is None: + continue + if catalog_band_priority(star.get('mag_band'), observed_filter) != 0: + continue + candidates.append({ + 'label': label, + 'star': star, + 'magnitude': float(magnitude), + 'magnitude_error': float(magnitude_error), + 'band_priority': 0, + }) + if not candidates: return None - - flats_img_list = [fits.getdata(flat_file) for flat_file in flat_files] - master_flat = np.median(flats_img_list, axis=0) - # Bias subtract after creating master flat - if master_bias is not None: - master_flat = master_flat - master_bias - # Normalize - medi = np.median(master_flat) - return master_flat / medi + candidates.sort(key=lambda candidate: ( + candidate['band_priority'], + candidate['magnitude_error'], + )) + return candidates[0] + + +def stellar_variability_catalog_profile( + catalog_match, observed_filter=None, gaia_lookup_state=None): + if not isinstance(catalog_match, dict): + return {} + magnitude = _finite_float(catalog_match.get('mag')) + magnitude_error = normalized_magnitude_error(catalog_match.get('error')) + color = nextastro_catalog_color_with_gaia_fallback( + catalog_match.get('catalog_row'), + observed_filter, + lookup_state=gaia_lookup_state, + ) + return { + 'magnitude': float(magnitude) if magnitude is not None else None, + 'magnitude_error': float(magnitude_error) if magnitude_error is not None else None, + 'magnitude_band': catalog_match.get('mag_band'), + 'color': _finite_float((color or {}).get('color')), + 'color_label': (color or {}).get('label'), + 'catalog_ra': _finite_float(catalog_match.get('catalog_ra')), + 'catalog_dec': _finite_float(catalog_match.get('catalog_dec')), + 'source_id': catalog_match.get('source_id'), + 'catalog_id': catalog_match.get('id'), + } -def convert_jd_to_bjd(non_bjd, p_dict, info_dict): - try: - goodTimes = JDUTC_to_BJDTDB(non_bjd, ra=p_dict['ra'], dec=p_dict['dec'], lat=info_dict['lat'], - longi=info_dict['long'], alt=info_dict['elev'])[0] - except: - targetloc = SkyCoord(p_dict['ra'], p_dict['dec'], unit=(u.deg, u.deg), frame='icrs') - obsloc = EarthLocation(lat=info_dict['lat'], lon=info_dict['long'], height=info_dict['elev']) - timesToConvert = Time(non_bjd, format='jd', scale='utc', location=obsloc) - ltt_bary = timesToConvert.light_travel_time(targetloc) - time_barycentre = timesToConvert.tdb + ltt_bary - goodTimes = time_barycentre.value - return goodTimes +def add_stellar_variability_member_similarity( + candidate, target_profile, observed_filter=None, gaia_lookup_state=None): + enriched = dict(candidate) + member_color = nextastro_catalog_color_with_gaia_fallback( + enriched.get('star', {}).get('catalog_row'), + observed_filter, + lookup_state=gaia_lookup_state, + ) + member_color_value = _finite_float((member_color or {}).get('color')) + target_color_value = _finite_float((target_profile or {}).get('color')) + member_color_label = (member_color or {}).get('label') + target_color_label = (target_profile or {}).get('color_label') + if ( + member_color_value is not None + and target_color_value is not None + and normalize_colour_index_label(member_color_label) + == normalize_colour_index_label(target_color_label) + ): + color_delta = abs(member_color_value - target_color_value) + else: + color_delta = None + + target_magnitude = _finite_float((target_profile or {}).get('magnitude')) + member_magnitude = _finite_float(enriched.get('magnitude')) + magnitude_delta = ( + abs(member_magnitude - target_magnitude) + if member_magnitude is not None and target_magnitude is not None + else None + ) + similarity_score = ( + float(np.hypot(color_delta, magnitude_delta)) + if color_delta is not None and magnitude_delta is not None + else None + ) + enriched.update({ + 'color': member_color_value, + 'color_label': member_color_label, + 'target_color': target_color_value, + 'target_color_label': target_color_label, + 'color_delta': color_delta, + 'target_magnitude': target_magnitude, + 'magnitude_delta': magnitude_delta, + 'color_magnitude_similarity_score': similarity_score, + }) + return enriched + + +def stellar_variability_ensemble_calibration_error_clip( + member_candidates, + sigma=STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_SIGMA, + min_members=STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS): + candidates = list(member_candidates or []) + errors = np.asarray( + [candidate.get('magnitude_error', np.nan) for candidate in candidates], + dtype=float, + ) + keep = np.isfinite(errors) & (errors > 0) + threshold = np.nan + center = np.nan + scatter = np.nan + if np.count_nonzero(keep) < 3: + return keep, { + 'center': center, + 'scatter': scatter, + 'high_threshold': threshold, + 'minimum_high_threshold': ( + STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_HIGH_THRESHOLD_FLOOR_MAG + ), + } + for _ in range(10): + kept_errors = errors[keep] + if kept_errors.size < 3: + break + center = float(bn.nanmedian(kept_errors)) + mad = float(bn.nanmedian(np.abs(kept_errors - center))) + robust_error_scatter = 1.4826 * mad if np.isfinite(mad) else np.nan + scatter_floor = max( + STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_FLOOR, + abs(center) * STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_FLOOR_FRACTION, + ) + scatter = max( + robust_error_scatter if np.isfinite(robust_error_scatter) else 0.0, + scatter_floor, + ) + threshold = max( + center + (float(sigma) * scatter), + STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_HIGH_THRESHOLD_FLOOR_MAG, + ) + updated_keep = keep & (errors <= threshold) + if np.count_nonzero(updated_keep) < int(min_members) or np.array_equal(updated_keep, keep): + break + keep = updated_keep + + return keep, { + 'center': center, + 'scatter': scatter, + 'high_threshold': threshold, + 'minimum_high_threshold': ( + STELLAR_VARIABILITY_ENSEMBLE_CALIBRATION_ERROR_HIGH_THRESHOLD_FLOOR_MAG + ), + } -def calculate_variablility(fit_lc_ref, fit_lc_best): - info_ref = None - mask_oot_ref = (fit_lc_ref.transit == 1) - mask_oot_best = (fit_lc_best.transit == 1) +def stellar_variability_acquisition_gap_boundaries( + times, + minimum_gap_ratio=STELLAR_VARIABILITY_COMPARISON_GAP_MIN_RATIO, + minimum_gap_seconds=STELLAR_VARIABILITY_COMPARISON_GAP_MIN_SECONDS, + minimum_side_points=STELLAR_VARIABILITY_COMPARISON_GAP_MIN_POINTS): + values = np.asarray(times, dtype=float).reshape(-1) + if values.size < (2 * int(minimum_side_points)): + return [] + differences = np.diff(values) + positive = differences[np.isfinite(differences) & (differences > 0)] + if positive.size == 0: + return [] + cadence_days = float(bn.nanmedian(positive)) + threshold_days = max( + float(minimum_gap_ratio) * cadence_days, + float(minimum_gap_seconds) / 86400.0, + ) + boundaries = [] + for boundary_index in np.flatnonzero( + np.isfinite(differences) & (differences >= threshold_days)) + 1: + if ( + boundary_index < int(minimum_side_points) + or values.size - boundary_index < int(minimum_side_points) + ): + continue + boundaries.append({ + 'source_index': int(boundary_index), + 'pre_time': float(values[boundary_index - 1]), + 'post_time': float(values[boundary_index]), + 'gap_seconds': float(differences[boundary_index - 1] * 86400.0), + 'median_cadence_seconds': float(cadence_days * 86400.0), + }) + return boundaries + + +def _robust_median_scatter_count(values): + finite = np.asarray(values, dtype=float) + finite = finite[np.isfinite(finite)] + if finite.size == 0: + return np.nan, np.nan, 0 + median = float(bn.nanmedian(finite)) + scatter = float(1.4826 * bn.nanmedian(np.abs(finite - median))) + return median, scatter, int(finite.size) + + +def stellar_variability_comparison_gap_stability( + candidates, + comp_flux_map, + times, + window_frames=STELLAR_VARIABILITY_COMPARISON_GAP_WINDOW_FRAMES, + minimum_points=STELLAR_VARIABILITY_COMPARISON_GAP_MIN_POINTS, + maximum_step_magnitude=STELLAR_VARIABILITY_COMPARISON_GAP_MAX_STEP_MAG, + minimum_significance=STELLAR_VARIABILITY_COMPARISON_GAP_MIN_SIGNIFICANCE): + candidates = list(candidates or []) + boundaries = stellar_variability_acquisition_gap_boundaries( + times, + minimum_side_points=minimum_points, + ) + summary = { + 'applied': False, + 'reason': None, + 'boundaries': boundaries, + 'window_frames': int(window_frames), + 'minimum_points_per_side': int(minimum_points), + 'maximum_allowed_step_magnitude': float(maximum_step_magnitude), + 'minimum_rejection_significance': float(minimum_significance), + 'candidates': {}, + } + if len(candidates) < 3: + summary['reason'] = 'fewer than three independently calibrated comparison candidates' + return summary + if not boundaries: + summary['reason'] = 'no acquisition gap exceeded the cadence-based threshold' + return summary + + times_array = np.asarray(times, dtype=float).reshape(-1) + instrumental_columns = [] + usable_candidates = [] + for candidate in candidates: + key = candidate.get('key') + flux = np.asarray(comp_flux_map.get(key, []), dtype=float).reshape(-1) + if flux.shape != times_array.shape: + continue + instrumental = np.full(flux.shape, np.nan, dtype=float) + valid = np.isfinite(flux) & (flux > 0) + with np.errstate(divide='ignore', invalid='ignore'): + instrumental[valid] = -2.5 * np.log10(flux[valid]) + center = bn.nanmedian(instrumental) + if not np.isfinite(center): + continue + instrumental_columns.append(instrumental - center) + usable_candidates.append(candidate) + + if len(usable_candidates) < 3: + summary['reason'] = 'fewer than three comparison candidates had usable flux series' + return summary + + instrumental_stack = np.column_stack(instrumental_columns) + window = max(int(window_frames), int(minimum_points)) + for candidate_index, candidate in enumerate(usable_candidates): + other_stack = np.delete(instrumental_stack, candidate_index, axis=1) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', category=RuntimeWarning) + leave_one_out_reference = np.nanmedian(other_stack, axis=1) + residual = instrumental_stack[:, candidate_index] - leave_one_out_reference + boundary_results = [] + rejected = False + maximum_absolute_step = 0.0 + maximum_step_significance = 0.0 + for boundary in boundaries: + boundary_index = boundary['source_index'] + pre = residual[max(0, boundary_index - window):boundary_index] + post = residual[boundary_index:min(residual.size, boundary_index + window)] + pre_median, pre_scatter, pre_count = _robust_median_scatter_count(pre) + post_median, post_scatter, post_count = _robust_median_scatter_count(post) + if pre_count < int(minimum_points) or post_count < int(minimum_points): + continue + step = float(post_median - pre_median) + uncertainty = float(np.hypot( + pre_scatter / np.sqrt(pre_count), + post_scatter / np.sqrt(post_count), + )) + significance = ( + float(abs(step) / uncertainty) + if np.isfinite(uncertainty) and uncertainty > 0 + else (float('inf') if step != 0 else 0.0) + ) + step_rejected = ( + abs(step) > float(maximum_step_magnitude) + and significance >= float(minimum_significance) + ) + rejected = rejected or step_rejected + maximum_absolute_step = max(maximum_absolute_step, abs(step)) + maximum_step_significance = max(maximum_step_significance, significance) + boundary_results.append({ + **boundary, + 'step_magnitude': step, + 'step_uncertainty_magnitude': uncertainty, + 'step_significance': significance, + 'pre_point_count': pre_count, + 'post_point_count': post_count, + 'rejected': step_rejected, + }) + summary['candidates'][candidate.get('key')] = { + 'label': candidate.get('label'), + 'position': candidate.get('position'), + 'catalog_source': candidate.get('star', {}).get('catalog_source'), + 'catalog_magnitude_band': candidate.get('star', {}).get('mag_band'), + 'maximum_absolute_step_magnitude': maximum_absolute_step, + 'maximum_step_significance': maximum_step_significance, + 'rejected': rejected, + 'boundary_results': boundary_results, + } - intx_times = np.intersect1d(fit_lc_best.jd_times[mask_oot_best], fit_lc_ref.jd_times[mask_oot_ref]) + summary['applied'] = True + return summary + + +def select_stellar_variability_ensemble_members( + ranked_summaries, + calibration_stars, + comp_flux_map, + observed_filter=None, + target_catalog_match=None, + max_members=STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS, + min_members=STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS, + times=None): + gaia_lookup_state = { + 'remaining': NEXTASTRO_GAIA_COLOR_LOOKUP_MAX_PER_SELECTOR, + 'attempted': 0, + 'matched': 0, + } + target_profile = stellar_variability_catalog_profile( + target_catalog_match, + observed_filter, + gaia_lookup_state=gaia_lookup_state, + ) + candidates = [] + rejected = [] + represented_catalog_identities = set() + for summary in ranked_summaries or []: + ckey = summary.get('key') + position = summary.get('position') + if not ckey or ckey not in comp_flux_map: + rejected.append({'key': ckey, 'reason': 'no usable photometry series'}) + continue + calibration = stellar_variability_calibration_for_position( + calibration_stars, + position, + observed_filter=observed_filter, + ) + if calibration is None: + rejected.append({'key': ckey, 'reason': 'no usable catalog calibration'}) + continue + catalog_identity = calibration_catalog_identity( + calibration.get('star'), + calibration.get('label'), + ) + if catalog_identity is not None and catalog_identity in represented_catalog_identities: + rejected.append({ + 'key': ckey, + 'reason': 'duplicate catalog source already represented by another ensemble candidate', + }) + continue - if intx_times.any(): - mask_ref = np.isin(fit_lc_ref.jd_times, intx_times) - mask_best = np.isin(fit_lc_best.jd_times, intx_times) + flux_values = np.asarray(comp_flux_map[ckey], dtype=float) + valid_flux = np.isfinite(flux_values) & (flux_values > 0) + if np.count_nonzero(valid_flux) < LIGHTCURVE_MIN_VALID_POINTS: + rejected.append({'key': ckey, 'reason': 'too few finite positive flux measurements'}) + continue + median_flux = float(bn.nanmedian(flux_values[valid_flux])) + if not np.isfinite(median_flux) or median_flux <= 0: + rejected.append({'key': ckey, 'reason': 'invalid median brightness'}) + continue - norm_flux_ref = (fit_lc_ref.data / np.nanmedian(fit_lc_ref.data[mask_ref]))[mask_ref] - norm_flux_best = (fit_lc_best.data / np.nanmedian(fit_lc_best.data[mask_best]))[mask_best] + candidates.append(add_stellar_variability_member_similarity( + { + 'key': ckey, + 'comp_index': summary.get('comp_index'), + 'label': summary.get('label', ckey), + 'position': position, + 'summary': summary, + 'median_flux': median_flux, + **calibration, + }, + target_profile, + observed_filter=observed_filter, + gaia_lookup_state=gaia_lookup_state, + )) + if catalog_identity is not None: + represented_catalog_identities.add(catalog_identity) + + gap_stability = stellar_variability_comparison_gap_stability( + candidates, + comp_flux_map, + times, + ) if times is not None else { + 'applied': False, + 'reason': 'observation times were unavailable', + 'boundaries': [], + 'candidates': {}, + } + if gap_stability.get('applied'): + stable_candidates = [] + for candidate in candidates: + diagnostic = gap_stability.get('candidates', {}).get(candidate.get('key'), {}) + candidate['gap_stability'] = diagnostic + candidate['gap_stability_max_abs_step_mag'] = diagnostic.get( + 'maximum_absolute_step_magnitude' + ) + candidate['gap_stability_max_significance'] = diagnostic.get( + 'maximum_step_significance' + ) + if diagnostic.get('rejected'): + rejected.append({ + 'key': candidate.get('key'), + 'label': candidate.get('label'), + 'position': candidate.get('position'), + 'catalog_source': candidate.get('star', {}).get('catalog_source'), + 'catalog_magnitude_band': candidate.get('star', {}).get('mag_band'), + 'reason': ( + 'comparison changed discontinuously relative to the leave-one-out comparison ' + 'ensemble across an acquisition gap' + ), + 'maximum_absolute_step_magnitude': diagnostic.get( + 'maximum_absolute_step_magnitude' + ), + 'maximum_step_significance': diagnostic.get('maximum_step_significance'), + 'boundary_results': diagnostic.get('boundary_results', []), + }) + continue + stable_candidates.append(candidate) + candidates = stable_candidates - info_ref = { - 'fit_lc': fit_lc_ref, - 'mask_ref': mask_ref, - 'res': norm_flux_best - norm_flux_ref, - } + candidates.sort(key=lambda candidate: (-candidate['median_flux'], candidate.get('comp_index', np.inf))) + try: + minimum_members = max(1, int(min_members)) + except (TypeError, ValueError): + minimum_members = STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS + keep_mask, clip_summary = stellar_variability_ensemble_calibration_error_clip( + candidates, + min_members=minimum_members, + ) + clipped_members = [] + for candidate, keep in zip(candidates, keep_mask): + if keep: + clipped_members.append(candidate) + else: + rejected.append({ + 'key': candidate.get('key'), + 'reason': ( + 'catalog magnitude uncertainty exceeded the stellar-variability ensemble ' + 'high-side sigma-clip threshold' + ), + 'magnitude_error': candidate.get('magnitude_error'), + }) - return info_ref + try: + member_limit = max(int(max_members), minimum_members) + except (TypeError, ValueError): + member_limit = STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS + prelimit_member_count = len(clipped_members) + members = list(clipped_members) + if len(members) > member_limit: + members.sort(key=lambda candidate: ( + _finite_float(candidate.get('gap_stability_max_abs_step_mag'), np.inf), + 0 if _finite_float(candidate.get('color_magnitude_similarity_score')) is not None else 1, + _finite_float(candidate.get('color_magnitude_similarity_score'), np.inf), + _finite_float(candidate.get('color_delta'), np.inf), + _finite_float(candidate.get('magnitude_delta'), np.inf), + -candidate.get('median_flux', 0.0), + candidate.get('comp_index', np.inf), + )) + excluded_by_limit = members[member_limit:] + members = members[:member_limit] + for candidate in excluded_by_limit: + rejected.append({ + 'key': candidate.get('key'), + 'reason': ( + f'not among the {member_limit} comparison stars closest to the target in ' + 'catalog color and magnitude' + ), + 'color_delta': candidate.get('color_delta'), + 'magnitude_delta': candidate.get('magnitude_delta'), + 'color_magnitude_similarity_score': candidate.get('color_magnitude_similarity_score'), + }) + for selection_rank, member in enumerate(members, start=1): + member['selection_rank'] = selection_rank + + log_nextastro_gaia_color_lookup_summary(gaia_lookup_state) + return { + 'members': members, + 'rejected': rejected, + 'calibration_error_clip': clip_summary, + 'target_catalog_profile': target_profile, + 'prelimit_member_count': prelimit_member_count, + 'member_limit': member_limit, + 'gap_stability': gap_stability, + } -def choose_comp_star_variability(fit_lc_refs, fit_lc_best, ref_comp, comp_stars, vsp_comp_stars, save): - colors = ["firebrick", "darkorange", "olivedrab", "lightseagreen", "steelblue", "rebeccapurple", "mediumvioletred"] - markers = ['.', 'v', 's', 'D', '^'] - k = 0 +def build_stellar_variability_calibrated_ensemble_series(target_flux, target_flux_error, + comp_flux_map, comp_error_map, members, + minimum_members=STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS, + validity_mask_func=valid_comparison_frame_mask): + target_flux = np.asarray(target_flux, dtype=float) + if target_flux.ndim != 1: + target_flux = target_flux.reshape(-1) + frame_count = target_flux.size + if target_flux_error is None: + target_flux_error = np.sqrt(np.clip(np.abs(target_flux), 1.0, None)) + target_flux_error = np.asarray(target_flux_error, dtype=float).reshape(-1) + if target_flux_error.shape != target_flux.shape: + target_flux_error = np.sqrt(np.clip(np.abs(target_flux), 1.0, None)) + + zero_points = [] + zero_point_errors = [] + raw_member_keys = [] + raw_comp_error_map = {} + magnitude_factor = 2.5 / np.log(10.0) + for member in members or []: + ckey = member.get('key') + if ckey not in comp_flux_map: + continue + comp_flux = np.asarray(comp_flux_map[ckey], dtype=float).reshape(-1) + if comp_flux.shape != target_flux.shape: + continue + comp_flux_error = None + if isinstance(comp_error_map, dict) and ckey in comp_error_map: + comp_flux_error = np.asarray(comp_error_map[ckey], dtype=float).reshape(-1) + if comp_flux_error is None or comp_flux_error.shape != comp_flux.shape: + comp_flux_error = np.sqrt(np.clip(np.abs(comp_flux), 1.0, None)) + raw_member_keys.append(ckey) + raw_comp_error_map[ckey] = comp_flux_error + + member_keep_mask = np.asarray( + member.get('summary', {}).get('ensemble_frame_keep_mask', np.ones(frame_count, dtype=bool)), + dtype=bool, + ) + if member_keep_mask.shape != target_flux.shape: + member_keep_mask = np.ones(frame_count, dtype=bool) + valid = ( + member_keep_mask + & np.isfinite(comp_flux) + & (comp_flux > 0) + & np.isfinite(comp_flux_error) + & (comp_flux_error >= 0) + ) + zero_point = np.full(target_flux.shape, np.nan, dtype=float) + zero_point_error = np.full(target_flux.shape, np.nan, dtype=float) + with np.errstate(divide='ignore', invalid='ignore'): + zero_point[valid] = member['magnitude'] + (2.5 * np.log10(comp_flux[valid])) + instrumental_error = magnitude_factor * comp_flux_error[valid] / comp_flux[valid] + zero_point_error[valid] = np.hypot(member['magnitude_error'], instrumental_error) + zero_points.append(zero_point) + zero_point_errors.append(zero_point_error) + + raw_reference_flux = np.full(target_flux.shape, np.nan, dtype=float) + raw_reference_flux_error = np.full(target_flux.shape, np.nan, dtype=float) + built_raw_reference, built_raw_member_keys = build_absolute_comp_ensemble_flux( + comp_flux_map, + raw_member_keys, + validity_mask_func=validity_mask_func, + ) + if built_raw_reference is not None and built_raw_member_keys == raw_member_keys: + built_raw_reference = np.asarray(built_raw_reference, dtype=float) + if built_raw_reference.shape == target_flux.shape: + raw_reference_flux = built_raw_reference + built_raw_reference_error = build_absolute_comp_ensemble_uncertainty( + comp_flux_map, + raw_comp_error_map, + raw_member_keys, + validity_mask_func=validity_mask_func, + ) + if ( + built_raw_reference_error is not None + and np.asarray(built_raw_reference_error).shape == target_flux.shape + ): + raw_reference_flux_error = np.asarray(built_raw_reference_error, dtype=float) - labels = {tuple(value['pos']): key for key, value in vsp_comp_stars.items()} + selected_member_count = len(members or []) + try: + minimum_required_members = max(1, int(minimum_members)) + except (TypeError, ValueError): + minimum_required_members = STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS + # An ensemble has a fixed membership. Allowing the contributing subset to + # change from frame to frame changes the photometric zero point and can + # create false variability. Every selected member must therefore be valid + # for a frame, or that frame is rejected. + required_members = max(minimum_required_members, selected_member_count) + member_text = "comparison star" if required_members == 1 else "comparison stars" + empty = { + 'applied': False, + 'failure_reason': ( + f'fewer than {required_members} calibrated {member_text} were usable for the reference.' + ), + 'magnitude': np.full(target_flux.shape, np.nan, dtype=float), + 'magnitude_error': np.full(target_flux.shape, np.nan, dtype=float), + 'relative_flux': np.full(target_flux.shape, np.nan, dtype=float), + 'relative_flux_error': np.full(target_flux.shape, np.nan, dtype=float), + 'synthetic_reference_flux': np.full(target_flux.shape, np.nan, dtype=float), + 'synthetic_reference_flux_error': np.full(target_flux.shape, np.nan, dtype=float), + 'raw_reference_flux': raw_reference_flux, + 'raw_reference_flux_error': raw_reference_flux_error, + 'valid_member_count': np.zeros(target_flux.shape, dtype=int), + } + if len(zero_points) < required_members: + return empty + + zero_point_stack = np.vstack(zero_points) + zero_point_error_stack = np.vstack(zero_point_errors) + valid_member = ( + np.isfinite(zero_point_stack) + & np.isfinite(zero_point_error_stack) + & (zero_point_error_stack > 0) + ) + valid_member_count = np.count_nonzero(valid_member, axis=0) + weights = np.zeros(zero_point_error_stack.shape, dtype=float) + weights[valid_member] = 1.0 / (zero_point_error_stack[valid_member] ** 2) + weight_sum = np.sum(weights, axis=0) + weighted_zero_point_sum = np.nansum(weights * zero_point_stack, axis=0) + valid_target = ( + np.isfinite(target_flux) + & (target_flux > 0) + & np.isfinite(target_flux_error) + & (target_flux_error >= 0) + ) + valid = ( + valid_target + & (valid_member_count >= required_members) + & np.isfinite(weight_sum) + & (weight_sum > 0) + ) + if np.count_nonzero(valid) < LIGHTCURVE_MIN_VALID_POINTS: + empty['valid_member_count'] = valid_member_count + empty['failure_reason'] = ( + f'too few frames retained at least {required_members} calibrated {member_text}.' + ) + return empty + + magnitude = np.full(target_flux.shape, np.nan, dtype=float) + magnitude_error = np.full(target_flux.shape, np.nan, dtype=float) + with np.errstate(divide='ignore', invalid='ignore'): + ensemble_zero_point = weighted_zero_point_sum[valid] / weight_sum[valid] + target_instrumental_error = magnitude_factor * target_flux_error[valid] / target_flux[valid] + magnitude[valid] = ensemble_zero_point - (2.5 * np.log10(target_flux[valid])) + magnitude_error[valid] = np.hypot(np.sqrt(1.0 / weight_sum[valid]), target_instrumental_error) + + baseline_magnitude = float(bn.nanmedian(magnitude[valid])) + relative_flux = np.full(target_flux.shape, np.nan, dtype=float) + relative_flux_error = np.full(target_flux.shape, np.nan, dtype=float) + synthetic_reference_flux = np.full(target_flux.shape, np.nan, dtype=float) + synthetic_reference_flux_error = np.full(target_flux.shape, np.nan, dtype=float) + with np.errstate(over='ignore', divide='ignore', invalid='ignore'): + relative_flux[valid] = 10.0 ** (-0.4 * (magnitude[valid] - baseline_magnitude)) + relative_flux_error[valid] = ( + relative_flux[valid] * (np.log(10.0) / 2.5) * magnitude_error[valid] + ) + synthetic_reference_flux[valid] = target_flux[valid] / relative_flux[valid] + target_fractional_error = target_flux_error[valid] / target_flux[valid] + ratio_fractional_error = relative_flux_error[valid] / relative_flux[valid] + reference_fractional_error = np.sqrt( + np.maximum((ratio_fractional_error ** 2) - (target_fractional_error ** 2), 0.0) + ) + synthetic_reference_flux_error[valid] = ( + synthetic_reference_flux[valid] * reference_fractional_error + ) - for i, ckey in enumerate(fit_lc_refs.keys()): - if i >= len(colors): - i = 0 - if k >= len(markers): - k = 0 - ref_comp[ckey] = calculate_variablility(fit_lc_refs[ckey]['myfit'], fit_lc_best) + return { + 'applied': True, + 'failure_reason': None, + 'magnitude': magnitude, + 'magnitude_error': magnitude_error, + 'relative_flux': relative_flux, + 'relative_flux_error': relative_flux_error, + 'synthetic_reference_flux': synthetic_reference_flux, + 'synthetic_reference_flux_error': synthetic_reference_flux_error, + 'raw_reference_flux': raw_reference_flux, + 'raw_reference_flux_error': raw_reference_flux_error, + 'valid_member_count': valid_member_count, + 'baseline_magnitude': baseline_magnitude, + } - if ref_comp[ckey]: - plt.errorbar(ref_comp[ckey]['fit_lc'].jd_times[ref_comp[ckey]['mask_ref']], ref_comp[ckey]['res'], - fmt=markers[k], color=colors[i], label=f"{labels[tuple(fit_lc_refs[ckey]['pos'])]}") - k += 1 - plot_variable_residuals(save) +def annotate_stellar_variability_ensemble_differential_photometry(lc_fit, ensemble_series): + """Attach the real ensemble flux scale without replacing normalized fitting inputs.""" + fit_shape = np.asarray(getattr(lc_fit, 'data', []), dtype=float).shape + source_indices = np.asarray( + getattr(lc_fit, 'stellar_variability_source_indices', []), + dtype=int, + ) + raw_reference_flux = np.asarray( + ensemble_series.get('raw_reference_flux', []), + dtype=float, + ) + raw_reference_flux_error = np.asarray( + ensemble_series.get('raw_reference_flux_error', []), + dtype=float, + ) + target_flux = np.asarray( + getattr(lc_fit, 'stellar_variability_target_flux', []), + dtype=float, + ) + target_flux_error = np.asarray( + getattr(lc_fit, 'stellar_variability_target_flux_error', []), + dtype=float, + ) - std_devs = {key: np.std(value['res']) for key, value in ref_comp.items() if value} - min_std_dev = min(std_devs, key=lambda y: abs(std_devs[y])) + if source_indices.shape != fit_shape or target_flux.shape != fit_shape: + raise RuntimeError( + "Calibrated comparison-ensemble fit did not retain aligned target fluxes for " + "differential-magnitude output." + ) + if ( + raw_reference_flux.ndim != 1 + or source_indices.size == 0 + or np.any(source_indices < 0) + or np.any(source_indices >= raw_reference_flux.size) + ): + raise RuntimeError( + "Raw comparison-ensemble reference flux is unavailable; refusing to report a " + "median-normalized light curve as differential magnitude." + ) + selected_reference_flux = raw_reference_flux[source_indices] + selected_reference_error = ( + raw_reference_flux_error[source_indices] + if raw_reference_flux_error.shape == raw_reference_flux.shape + else np.full(fit_shape, np.nan, dtype=float) + ) + attached = annotate_differential_magnitude_raw_photometry( + lc_fit, + target_flux, + selected_reference_flux, + target_flux_error=target_flux_error, + reference_flux_error=selected_reference_error, + ) + if not attached: + raise RuntimeError( + "Raw target and comparison-ensemble fluxes could not be aligned for " + "differential-magnitude output." + ) + return lc_fit + + +def stellar_variability_json_safe(value): + if isinstance(value, dict): + return {str(key): stellar_variability_json_safe(subvalue) for key, subvalue in value.items()} + if isinstance(value, (list, tuple)): + return [stellar_variability_json_safe(item) for item in value] + if isinstance(value, np.ndarray): + return stellar_variability_json_safe(value.tolist()) + if isinstance(value, np.generic): + return stellar_variability_json_safe(value.item()) + if isinstance(value, Path): + return str(value) + if isinstance(value, float): + return float(value) if np.isfinite(value) else None + if isinstance(value, (bool, int, str)) or value is None: + return value + return str(value) + + +def stellar_variability_ensemble_member_json(member): + star = member.get('star', {}) if isinstance(member, dict) else {} + return { + 'selection_rank': member.get('selection_rank'), + 'key': member.get('key'), + 'label': member.get('label'), + 'pixel_position': member.get('position'), + 'ra_deg': star.get('ra', star.get('catalog_ra')), + 'dec_deg': star.get('dec', star.get('catalog_dec')), + 'catalog_source': star.get('catalog_source'), + 'catalog_source_id': star.get('source_id'), + 'catalog_id': star.get('id'), + 'catalog_magnitude': member.get('magnitude'), + 'catalog_magnitude_error': member.get('magnitude_error'), + 'catalog_magnitude_band': star.get('mag_band'), + 'catalog_color': member.get('color'), + 'catalog_color_label': member.get('color_label'), + 'target_catalog_color': member.get('target_color'), + 'target_catalog_color_label': member.get('target_color_label'), + 'color_delta': member.get('color_delta'), + 'target_catalog_magnitude': member.get('target_magnitude'), + 'magnitude_delta': member.get('magnitude_delta'), + 'color_magnitude_similarity_score': member.get('color_magnitude_similarity_score'), + 'median_flux_adu': member.get('median_flux'), + 'overexposure_rejected_frame_count': member.get('summary', {}).get( + 'overexposure_rejected_count', 0 + ), + 'gap_stability': member.get('gap_stability'), + } - return comp_stars[min_std_dev] +def save_stellar_variability_ensemble_selection_json( + lc_fit, + save, + target_name, + observation_date=None, + target_metadata=None): + members = list(getattr(lc_fit, 'stellar_variability_ensemble_members', []) or []) + selection = getattr(lc_fit, 'stellar_variability_ensemble_selection', {}) or {} + if not members: + return None + target_profile = ( + selection.get('target_catalog_profile') + or getattr(lc_fit, 'stellar_variability_target_catalog_profile', {}) + or {} + ) + metadata = dict(target_metadata or {}) + metadata.setdefault('name', target_name) + metadata['catalog_profile'] = target_profile + maximum_members = selection.get( + 'member_limit', + STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS, + ) + prelimit_member_count = selection.get('prelimit_member_count', len(members)) + payload = { + 'target': metadata, + 'ensemble': { + 'selection_rule': ( + 'After saturation, VSX, coverage, stability, and high catalog-error rejection, ' + f'use at most {maximum_members} stars ranked by joint catalog color and magnitude ' + 'distance to the target; ' + 'retain a frame only when every selected ensemble member is valid.' + ), + 'minimum_members': STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS, + 'per_frame_required_members': len(members), + 'maximum_members': maximum_members, + 'member_count_before_limit': prelimit_member_count, + # Retained for consumers of EnsembleSelection JSON written before + # Retained after the fixed five-member limit became configurable. + 'member_count_before_five_star_limit': prelimit_member_count, + 'selected_member_count': len(members), + 'calibration_error_clip': selection.get( + 'calibration_error_clip', + getattr(lc_fit, 'stellar_variability_ensemble_calibration_error_clip', {}), + ), + 'members': [stellar_variability_ensemble_member_json(member) for member in members], + 'rejected_candidates': selection.get('rejected', []), + 'comparison_gap_stability': selection.get('gap_stability', {}), + }, + } + output_dir = Path(save) + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / safe_output_filename( + 'EnsembleSelection', + target_name, + filename_date_token(observation_date) if observation_date else 'undated', + extension='json', + ) + with output_path.open('w', encoding='utf-8') as handle: + json.dump(stellar_variability_json_safe(payload), handle, indent=2, sort_keys=True) + handle.write('\n') + return output_path -def stellar_variability(fit_lc_refs, fit_lc_best, comp_stars, vsp_comp_stars, vsp_ind, best_comp, save, s_name): - info_comps = {} +def save_stellar_variability_magnitude_csv(vsp_params, save, target_name, observation_date=None): + if not vsp_params: + return None + output_dir = Path(save) + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / safe_output_filename( + 'StellarVariability', + target_name, + filename_date_token(observation_date) if observation_date else 'undated', + extension='csv', + ) + with output_path.open('w', encoding='utf-8', newline='') as handle: + writer = csv.writer(handle) + writer.writerow([ + 'BJD_TDB', + 'Airmass', + 'Apparent Magnitude', + 'Apparent Magnitude Error', + 'Raw Differential Magnitude', + 'Raw Differential Magnitude Error', + 'Filter', + 'Comparison', + ]) + for row in vsp_params: + differential_mag = row.get( + 'differential_mag', + row.get('differential_magnitude'), + ) + differential_mag_err = row.get( + 'differential_mag_err', + row.get('differential_magnitude_error'), + ) + writer.writerow([ + row.get('time'), + row.get('airmass'), + row.get('mag'), + row.get('mag_err'), + differential_mag, + differential_mag_err, + row.get('mag_band') or row.get('observed_filter'), + row.get('cname'), + ]) + return output_path + + +def save_stellar_variability_differential_products( + lc_fit, + save, + target_name, + observation_date=None, + observed_filter=None): + """Save raw target/reference variability products without airmass detrending.""" + csv_path = None try: - if best_comp is None or (best_comp not in vsp_ind): - comp_pos = choose_comp_star_variability(fit_lc_refs, fit_lc_best, info_comps, comp_stars, vsp_comp_stars, - save) - else: - comp_pos = comp_stars[best_comp] - info_comps[best_comp] = calculate_variablility(fit_lc_refs[best_comp]['myfit'], fit_lc_best) - except Exception as e: - log_info(f"Error selecting or calculating variability for comparison star: {e}", warn=True) - return [] - + csv_path = write_differential_magnitude_csv( + lc_fit, + save, + target_name, + observation_date=observation_date, + observed_filter=observed_filter, + out_of_transit_only=True, + apply_airmass_correction=False, + filename_prefix='StellarVariabilityDifferentialMagnitude', + ) + except Exception as exc: + log_info( + "Warning: could not save the stellar-variability differential-magnitude CSV " + f"({describe_retry_exception(exc)}).", + warn=True, + ) try: - comp_star = next(vsp_comp_stars[ckey] for ckey in vsp_comp_stars.keys() if comp_pos == vsp_comp_stars[ckey]['pos']) - vsp_auid_comp = next(key for key, value in vsp_comp_stars.items() if value['pos'] == comp_pos) - except StopIteration: - log_info("Comparison star or VSP AUID not found.", warn=True) + plot_differential_magnitude( + lc_fit, + target_name, + save, + observation_date or 'undated', + observed_filter=observed_filter, + out_of_transit_only=True, + apply_airmass_correction=False, + filename_prefix='StellarVariabilityDifferentialMagnitude', + save_stellar_variability_alias=True, + ) + except Exception as exc: + log_info( + "Warning: could not save the stellar-variability differential-magnitude plot " + f"({describe_retry_exception(exc)}).", + warn=True, + ) + return csv_path + + +def build_stellar_variability_ensemble_params_from_fit( + lc_fit, + save, + s_name, + observed_filter=None, + observation_date=None, + target_metadata=None): + save_stellar_variability_differential_products( + lc_fit, + save, + s_name, + observation_date=observation_date, + observed_filter=observed_filter, + ) + magnitudes = np.asarray( + getattr(lc_fit, 'stellar_variability_ensemble_magnitudes', []), + dtype=float, + ) + magnitude_errors = np.asarray( + getattr(lc_fit, 'stellar_variability_ensemble_magnitude_errors', []), + dtype=float, + ) + # Public stellar-variability products are explicitly labelled BJD_TDB. The + # light-curve ``time`` array carries BJD_TDB, while ``jd_times`` preserves + # the original FITS JD/UTC timestamps for frame-level diagnostics. + times = np.asarray(getattr(lc_fit, 'time', getattr(lc_fit, 'jd_times', [])), dtype=float) + airmass = np.asarray(getattr(lc_fit, 'airmass', np.ones(times.shape)), dtype=float) + members = list(getattr(lc_fit, 'stellar_variability_ensemble_members', []) or []) + if not (magnitudes.shape == magnitude_errors.shape == times.shape == airmass.shape): + log_info("Warning: calibrated stellar-variability ensemble arrays had inconsistent shapes.", warn=True) return [] - try: - Mc, Mc_err = comp_star['mag'], comp_star['error'] - - info_comp = info_comps[comp_stars.index(comp_pos)] - lc_fit = info_comp['fit_lc'] - mask_ref = info_comp['mask_ref'] - - oot_scatter = np.std((lc_fit.data / lc_fit.airmass_model)[mask_ref]) - norm_flux_unc = oot_scatter * lc_fit.airmass_model[mask_ref] - norm_flux_unc /= np.nanmedian(lc_fit.data[mask_ref]) - - model = np.exp(lc_fit.parameters['a2'] * lc_fit.airmass_model[mask_ref]) - flux = lc_fit.data[mask_ref] - detrended = flux / model - - Mt = Mc - (2.5 * np.log10(detrended)) - Mt_err = (Mc_err ** 2 + (-2.5 * norm_flux_unc / (detrended * np.log(10))) ** 2) ** 0.5 - except KeyError as e: - log_info(f"Key error in processing stellar variability: {e}", warn=True) - return [] - except Exception as e: - log_info(f"Error in processing stellar variability: {e}", warn=True) + valid = ( + np.isfinite(times) + & np.isfinite(airmass) + & np.isfinite(magnitudes) + & np.isfinite(magnitude_errors) + & (magnitude_errors > 0) + & (magnitudes <= MAX_APPARENT_MAGNITUDE) + ) + if not np.any(valid): + log_info("Warning: calibrated stellar-variability ensemble produced no finite magnitude rows.", warn=True) return [] - try: - vsp_params = [{ - 'time': lc_fit.jd_times[mask_ref][i], - 'airmass': lc_fit.airmass[mask_ref][i], - 'mag': mt, - 'mag_err': Mt_err[i], - 'cname': vsp_auid_comp, - 'cmag': Mc, - 'pos': comp_pos - } for i, mt in enumerate(Mt)] - - plot_stellar_variability(vsp_params, save, s_name, vsp_auid_comp) - except Exception as e: - log_info(f"Error in plotting or finalizing stellar variability data: {e}", warn=True) - return [] + member_labels = [member.get('label') for member in members] + member_positions = [member.get('position') for member in members] + member_catalog_magnitudes = [member.get('magnitude') for member in members] + member_catalog_errors = [member.get('magnitude_error') for member in members] + member_catalog_sources = [member.get('star', {}).get('catalog_source') for member in members] + member_ra_degs = [] + member_dec_degs = [] + member_details = [] + for member in members: + star = member.get('star', {}) if isinstance(member, dict) else {} + ra_deg = _finite_float(star.get('ra', star.get('catalog_ra'))) + dec_deg = _finite_float(star.get('dec', star.get('catalog_dec'))) + member_ra_degs.append(ra_deg) + member_dec_degs.append(dec_deg) + member_details.append({ + 'selection_rank': member.get('selection_rank'), + 'key': member.get('key'), + 'label': member.get('label'), + 'ra_deg': ra_deg, + 'dec_deg': dec_deg, + 'pixel_position': member.get('position'), + 'catalog_magnitude': member.get('magnitude'), + 'catalog_magnitude_error': member.get('magnitude_error'), + 'catalog_magnitude_band': star.get('mag_band'), + 'catalog_source': star.get('catalog_source'), + 'catalog_source_id': star.get('source_id'), + 'catalog_id': star.get('id'), + }) + member_catalog_colors = [member.get('color') for member in members] + member_catalog_color_labels = [member.get('color_label') for member in members] + member_color_deltas = [member.get('color_delta') for member in members] + member_magnitude_deltas = [member.get('magnitude_delta') for member in members] + member_similarity_scores = [member.get('color_magnitude_similarity_score') for member in members] + display_label = f"ENSEMBLE ({len(members)} stars)" + catalog_mag_band = preferred_catalog_magnitude_band_for_filter(observed_filter) + measurement_mag_band = reported_stellar_variability_band( + observed_filter, + fallback_band=catalog_mag_band, + ) + differential_magnitudes = np.full(times.shape, np.nan, dtype=float) + differential_magnitude_errors = np.full(times.shape, np.nan, dtype=float) + differential_series = differential_magnitude_series_from_fit( + lc_fit, + apply_airmass_correction=False, + ) + if differential_series is not None: + differential_source_mask = np.asarray( + differential_series.get('source_mask', []), + dtype=bool, + ) + if differential_source_mask.shape == times.shape: + differential_magnitudes[differential_source_mask] = differential_series['magnitude'] + differential_magnitude_errors[differential_source_mask] = ( + differential_series['magnitude_error'] + ) + vsp_params = [] + for time_value, airmass_value, magnitude, magnitude_error, differential_mag, differential_mag_err in zip( + times[valid], + airmass[valid], + magnitudes[valid], + magnitude_errors[valid], + differential_magnitudes[valid], + differential_magnitude_errors[valid], + ): + vsp_params.append({ + 'time': time_value, + 'airmass': airmass_value, + 'mag': magnitude, + 'mag_err': magnitude_error, + 'differential_mag': differential_mag, + 'differential_mag_err': differential_mag_err, + 'cname': display_label, + 'cmag': None, + 'cmag_err': None, + 'pos': member_positions, + 'comp_ra': None, + 'comp_dec': None, + 'catalog_ra': None, + 'catalog_dec': None, + 'catalog_source': 'Calibrated comparison-star ensemble', + 'is_aavso_vsp': False, + 'mag_band': measurement_mag_band, + 'catalog_mag_band': catalog_mag_band, + 'observed_filter': observed_filter, + 'ensemble_reference': True, + 'ensemble_member_count': len(members), + 'ensemble_member_labels': member_labels, + 'ensemble_member_positions': member_positions, + 'ensemble_member_catalog_magnitudes': member_catalog_magnitudes, + 'ensemble_member_catalog_errors': member_catalog_errors, + 'ensemble_member_catalog_sources': member_catalog_sources, + 'ensemble_member_ra_degs': member_ra_degs, + 'ensemble_member_dec_degs': member_dec_degs, + 'ensemble_members': member_details, + 'ensemble_member_catalog_colors': member_catalog_colors, + 'ensemble_member_catalog_color_labels': member_catalog_color_labels, + 'ensemble_member_color_deltas': member_color_deltas, + 'ensemble_member_magnitude_deltas': member_magnitude_deltas, + 'ensemble_member_similarity_scores': member_similarity_scores, + }) + try: + lc_fit.stellar_variability_params = vsp_params + lc_fit.stellar_variability_target_name = s_name + lc_fit.stellar_variability_reference_label = display_label + except Exception: + pass + plot_stellar_variability(vsp_params, save, s_name, display_label) + save_stellar_variability_ensemble_selection_json( + lc_fit, + save, + s_name, + observation_date=observation_date, + target_metadata=target_metadata, + ) return vsp_params -# Mid-Transit Time Prior Helper Functions -def numberOfTransitsAway(timeData, period, originalT): - return int((np.nanmin(timeData) - originalT) / period) + 1 +def should_require_apparent_magnitudes(config_value): + return parse_bool_config_value( + config_value, + REQUIRE_APPARENT_MAGNITUDES_DEFAULT, + 'require_apparent_magnitudes', + ) -def nearestTransitTime(timeData, period, originalT): - nearT = ((numberOfTransitsAway(timeData, period, originalT) * period) + originalT) - return nearT +def should_use_exactly_the_comps_provided(config_value): + return parse_bool_config_value( + config_value, + USE_EXACTLY_PROVIDED_COMPARISONS_DEFAULT, + 'use_exactly_the_comps_provided', + ) -def save_comp_ra_dec(wcs_file, ra_file, dec_file, comp_coords): - comp_ra, comp_dec = None, None +def build_stellar_variability_params_from_photometry_selection( + selection, + calibration_stars, + save, + s_name, + observed_filter=None, + observation_date=None, + target_metadata=None): + selected_result = (selection or {}).get('selected_result') + if not selected_result or selected_result.get('fit') is None: + log_info( + "Warning: calibrated stellar-variability photometry did not select a usable " + "out-of-transit reference, so AID magnitude output could not be created.", + warn=True, + ) + return [] - if wcs_file: - comp_ra = ra_file[int(comp_coords[1])][int(comp_coords[0])] - comp_dec = dec_file[int(comp_coords[1])][int(comp_coords[0])] + selected_fit = selected_result['fit'] + if selected_result.get('comp_index') is None: + return build_stellar_variability_ensemble_params_from_fit( + selected_fit, + save, + s_name, + observed_filter=observed_filter, + observation_date=observation_date, + target_metadata=target_metadata, + ) - comp_star = { - 'ra': str(comp_ra) if comp_ra else comp_ra, - 'dec': str(comp_dec) if comp_dec else comp_dec, - 'x': str(comp_coords[0]) if comp_coords[0] else comp_coords[0], - 'y': str(comp_coords[1]) if comp_coords[1] else comp_coords[1] - } + selected_position = selected_result.get('position') + calibration = stellar_variability_calibration_for_position( + calibration_stars, + selected_position, + observed_filter=observed_filter, + ) + if calibration is None: + log_info( + "Warning: the fallback stellar-variability comparison star had no usable " + "same-band catalog calibration, so AID magnitude output could not be created.", + warn=True, + ) + return [] - return comp_star + return build_stellar_variability_params_from_fit( + selected_fit, + calibration['star'], + selected_position, + calibration['label'], + save, + s_name, + observed_filter=observed_filter, + observation_date=observation_date, + ) -def realTimeReduce(i, target_name, p_dict, info_dict, ax): - timeList, airMassList, exptimes, norm_flux = [], [], [], [] +def select_stellar_variability_only_photometry(times, jd_times, airmass, p_dict, comparison_calibration, + psf_data, aper_data, target_psf_flux, + psf_flux_data=None, + psf_noise_data=None, + plot_time_range=None, + use_adaptive_apertures=False, + adaptive_aperture_values=None, + adaptive_annulus_values=None, + fallback_sigma=np.nan, + exposure_times_seconds=None, + gain_e_per_adu=None, + use_ensemble_photometry=True, + maximum_number_of_ensemble_comparisons_for_stellar_variability= + STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS, + calibration_stars=None, + observed_filter=None, + target_catalog_match=None, + require_apparent_magnitudes=True, + use_exactly_the_comps_provided=False): + ranked_summaries = ranked_comparison_calibration_summaries( + comparison_calibration, + include_unvetted=use_exactly_the_comps_provided, + ) + if not ranked_summaries: + return { + 'ranked_summaries': [], + 'attempts': [], + 'selected_result': None, + 'selection_metric': 'stellar_variability_scatter', + } - plateStatus.initializeFilenames(info_dict['images']) - inputfiles = corruption_check(info_dict['images']) - # time sort images - times = [] - for ifile in inputfiles: - plateStatus.setCurrentFilename(ifile) - extension = 0 - header = fits.getheader(filename=ifile, ext=extension) - while header['NAXIS'] == 0: - extension += 1 - header = fits.getheader(filename=ifile, ext=extension) - obsTime = img_time_bjd_tdb(header, p_dict, info_dict) - times.append(obsTime) - plateStatus.setObsTime(obsTime) + method = comparison_calibration['method'] + method_label = comparison_calibration.get('method_label', method) + aperture_index = comparison_calibration.get('a') + annulus_index = comparison_calibration.get('an') + if method == 'psf': + frame_count = target_psf_flux.shape[0] + target_flux = np.asarray(target_psf_flux, dtype=float) + psf_flux_data = psf_flux_data_source(psf_data, psf_flux_data) + target_flux_error = ( + np.asarray(psf_noise_data.get('target'), dtype=float) + if isinstance(psf_noise_data, dict) and 'target' in psf_noise_data + else None + ) + else: + frame_count = aper_data['target'].shape[0] + target_flux = np.asarray(aper_data['target'][:, aperture_index, annulus_index], dtype=float) + target_flux_error = ( + np.asarray(aper_data['target_unc'][:, aperture_index, annulus_index], dtype=float) + if isinstance(aper_data, dict) and 'target_unc' in aper_data + else None + ) - si = np.argsort(times) - inputfiles = np.array(inputfiles)[si] - exotic_UIprevTPX = info_dict['tar_coords'][0] - exotic_UIprevTPY = info_dict['tar_coords'][1] + exposure_times_array = None if exposure_times_seconds is None else np.asarray(exposure_times_seconds, dtype=float) + if exposure_times_array is not None and exposure_times_array.shape != times.shape: + exposure_times_array = None + + adaptive_summary = build_comparison_candidate_adaptive_summary( + comparison_calibration, + psf_data, + use_adaptive_apertures=use_adaptive_apertures, + adaptive_aperture_values=adaptive_aperture_values, + adaptive_annulus_values=adaptive_annulus_values, + fallback_sigma=fallback_sigma, + ) + field_image_keep_mask = np.asarray( + comparison_calibration.get('field_image_keep_mask', np.ones(times.shape[0], dtype=bool)), + dtype=bool, + ) + if field_image_keep_mask.shape != times.shape: + field_image_keep_mask = np.ones(times.shape[0], dtype=bool) + field_image_clip_diagnostic = None + if np.any(~field_image_keep_mask): + required_pairs = comparison_calibration.get('image_outlier_required_valid_pairs', 0) + sigma_threshold = comparison_calibration.get('image_outlier_sigma', COMPARISON_IMAGE_OUTLIER_SIGMA) + field_image_clip_diagnostic = build_time_rejection_diagnostic( + "Comparison-field image clip", + times, + field_image_keep_mask, + note=( + "Dropped frames flagged after comparison-star suitability clipping because every " + f"valid pairwise comparison was more than {sigma_threshold:.2f} sigma from its flat-line median " + f"(min valid pair count={required_pairs})." + ), + ) - plateStatus.setCurrentFilename(inputfiles[0]) - wcs_file = check_wcs(inputfiles[0], info_dict['save'], info_dict['plate_opt'], rt=True) - comp_star = info_dict['comp_stars'] - tar_radec, comp_radec = None, [] + attempts = [] + if use_ensemble_photometry: + if method == 'psf': + comp_flux_map = {} + comp_error_map = {} + for summary in ranked_summaries: + ckey = summary.get('key') + if not ckey or ckey not in psf_flux_data: + continue + quality_mask = np.asarray( + summary.get( + 'psf_quality_keep_mask', + psf_quality_mask_for_key( + psf_data, + ckey, + frame_count, + psf_flux_data=psf_flux_data, + ), + ), + dtype=bool, + ) + comp_flux_map[ckey] = psf_flux_series_from_rows(psf_flux_data[ckey], quality_mask) + if isinstance(psf_noise_data, dict) and ckey in psf_noise_data: + comp_error_map[ckey] = mask_series_with_quality(psf_noise_data[ckey], quality_mask) + target_shape_mask = target_psf_shape_quality_mask( + target_psf_quality_rows(psf_data, psf_flux_data=psf_flux_data) + ) + if target_shape_mask.shape != times.shape: + target_shape_mask = np.ones(times.shape[0], dtype=bool) + candidate_target_flux = mask_series_with_quality(target_flux, target_shape_mask) + candidate_target_flux_error = ( + None + if target_flux_error is None + else mask_series_with_quality(target_flux_error, target_shape_mask) + ) + else: + comp_flux_map = {} + comp_error_map = {} + for summary in ranked_summaries: + ckey = summary.get('key') + if not ckey or ckey not in aper_data: + continue + quality_mask = np.asarray( + summary.get( + 'psf_quality_keep_mask', + psf_quality_mask_for_key(psf_data, ckey, frame_count), + ), + dtype=bool, + ) + comp_flux_map[ckey] = mask_series_with_quality( + aper_data[ckey][:, aperture_index, annulus_index], + quality_mask, + ) + error_key = f"{ckey}_unc" + if error_key in aper_data: + comp_error_map[ckey] = mask_series_with_quality( + aper_data[error_key][:, aperture_index, annulus_index], + quality_mask, + ) + target_shape_mask = np.ones(times.shape[0], dtype=bool) + candidate_target_flux = target_flux + candidate_target_flux_error = target_flux_error + + fixed_reference_mode = ( + use_exactly_the_comps_provided or not require_apparent_magnitudes + ) + prebuilt_ensemble_series = None + if fixed_reference_mode: + if use_exactly_the_comps_provided: + selected_summaries = list(ranked_summaries) + else: + selected_summaries = list(ranked_summaries)[ + :parse_maximum_number_of_ensemble_comparisons_for_stellar_variability( + maximum_number_of_ensemble_comparisons_for_stellar_variability + ) + ] + ensemble_members = [] + calibrated_members = [] + for summary in selected_summaries: + member = { + 'key': summary.get('key'), + 'comp_index': summary.get('comp_index'), + 'label': summary.get('label', summary.get('key')), + 'position': summary.get('position'), + 'summary': summary, + } + ensemble_members.append(member) + calibration = stellar_variability_calibration_for_position( + calibration_stars, + summary.get('position'), + observed_filter=observed_filter, + ) + if calibration is not None: + calibrated_members.append({**member, **calibration}) + + requested_keys = [member['key'] for member in ensemble_members if member.get('key')] + if len(calibrated_members) == len(ensemble_members) and ensemble_members: + prebuilt_ensemble_series = build_stellar_variability_calibrated_ensemble_series( + candidate_target_flux, + candidate_target_flux_error, + comp_flux_map, + comp_error_map, + calibrated_members, + minimum_members=len(calibrated_members), + validity_mask_func=( + robust_flux_floor_mask if method == 'psf' else valid_comparison_frame_mask + ), + ) + if prebuilt_ensemble_series.get('applied'): + ensemble_members = calibrated_members + if not prebuilt_ensemble_series or not prebuilt_ensemble_series.get('applied'): + relative_series = build_relative_comparison_ensemble_series( + candidate_target_flux, + candidate_target_flux_error, + comp_flux_map, + comp_error_map, + requested_keys, + validity_mask_func=( + robust_flux_floor_mask if method == 'psf' else valid_comparison_frame_mask + ), + ) + if relative_series.get('applied'): + valid_member_count = np.where( + relative_series['valid_mask'], + len(requested_keys), + 0, + ) + prebuilt_ensemble_series = { + 'applied': True, + 'failure_reason': None, + 'magnitude': np.full(candidate_target_flux.shape, np.nan, dtype=float), + 'magnitude_error': np.full(candidate_target_flux.shape, np.nan, dtype=float), + 'relative_flux': relative_series['relative_flux'], + 'relative_flux_error': relative_series['relative_flux_error'], + 'synthetic_reference_flux': relative_series['reference_flux'], + 'synthetic_reference_flux_error': relative_series['reference_flux_error'], + 'raw_reference_flux': relative_series['reference_flux'], + 'raw_reference_flux_error': relative_series['reference_flux_error'], + 'valid_member_count': valid_member_count, + 'baseline_magnitude': np.nan, + } + else: + prebuilt_ensemble_series = relative_series + member_selection = { + 'members': ensemble_members, + 'rejected': [], + 'calibration_error_clip': {}, + 'target_catalog_profile': {}, + 'prelimit_member_count': len(ensemble_members), + 'member_limit': len(ensemble_members), + 'gap_stability': { + 'applied': False, + 'reason': 'disabled for a fixed, unvetted comparison reference', + 'boundaries': [], + 'candidates': {}, + }, + 'fixed_reference': True, + 'apparent_calibration_available': bool( + prebuilt_ensemble_series + and prebuilt_ensemble_series.get('applied') + and np.any(np.isfinite(prebuilt_ensemble_series.get('magnitude', []))) + ), + } + else: + member_selection = select_stellar_variability_ensemble_members( + ranked_summaries, + calibration_stars, + comp_flux_map, + observed_filter=observed_filter, + target_catalog_match=target_catalog_match, + max_members=maximum_number_of_ensemble_comparisons_for_stellar_variability, + times=times, + ) + ensemble_members = member_selection['members'] + clip_summary = member_selection['calibration_error_clip'] + for rejected_member in member_selection['rejected']: + log_info( + "Stellar-variability ensemble excluded " + f"{rejected_member.get('label') or rejected_member.get('key') or 'comparison candidate'}: " + f"{rejected_member.get('reason')}." + ) - if wcs_file: - wcs_header = fits.getheader(filename=wcs_file) + if len(ensemble_members) >= STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS: + member_text = ", ".join( + ( + f"{member['label']} (catalog sigma={member['magnitude_error']:.4f} mag)" + if member.get('magnitude_error') is not None + else f"{member['label']} (differential only)" + ) + for member in ensemble_members + ) + threshold = clip_summary.get('high_threshold', np.nan) + threshold_text = ( + f"; high-side calibration-error clip threshold={threshold:.4f} mag" + if np.isfinite(threshold) + else "" + ) + log_info( + f"Using a {len(ensemble_members)}-star comparison ensemble for " + "stellar-variability products only: " + f"{member_text}{threshold_text}." + ) + if member_selection.get('prelimit_member_count', 0) > len(ensemble_members): + log_info( + "Stellar-variability ensemble had more than " + f"{member_selection.get('member_limit')} usable stars; retained the configured maximum " + f"of {len(ensemble_members)} closest " + "to the target in catalog color and magnitude." + ) + ensemble_series = prebuilt_ensemble_series or build_stellar_variability_calibrated_ensemble_series( + candidate_target_flux, + candidate_target_flux_error, + comp_flux_map, + comp_error_map, + ensemble_members, + validity_mask_func=( + robust_flux_floor_mask if method == 'psf' else valid_comparison_frame_mask + ), + ) + if ensemble_series.get('applied'): + fit_mask = ( + field_image_keep_mask + & target_shape_mask + & np.isfinite(ensemble_series['relative_flux']) + & np.isfinite(ensemble_series['relative_flux_error']) + & (ensemble_series['relative_flux'] > 0) + & (ensemble_series['relative_flux_error'] > 0) + ) + availability_diagnostic = build_time_rejection_diagnostic( + "Stellar-variability comparison ensemble availability filter", + times, + fit_mask, + note=( + "Kept frames with a finite target measurement and every selected comparison " + "ensemble member." + ), + ) + filter_diagnostics = [] + if field_image_clip_diagnostic is not None: + filter_diagnostics.append(field_image_clip_diagnostic) + if availability_diagnostic is not None: + filter_diagnostics.append(availability_diagnostic) + + fit_diagnostics = diagnose_lightcurve_fit_inputs( + times[fit_mask], + candidate_target_flux[fit_mask], + ensemble_series['synthetic_reference_flux'][fit_mask], + airmass[fit_mask], + target_flux_error=( + None + if candidate_target_flux_error is None + else candidate_target_flux_error[fit_mask] + ), + comp_flux_error=ensemble_series['synthetic_reference_flux_error'][fit_mask], + enforce_relative_flux_max=False, + expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), + ) + exposure_times_for_fit = ( + None + if exposure_times_array is None + else exposure_times_array[fit_mask] + ) + prepared = { + 'applied': True, + 'failure_reason': None, + 'time': times[fit_mask], + 'flux': ensemble_series['relative_flux'][fit_mask], + 'unc': ensemble_series['relative_flux_error'][fit_mask], + 'airmass': airmass[fit_mask], + 'jd_time': jd_times[fit_mask], + 'exposure_time_seconds': exposure_times_for_fit, + 'target_flux': candidate_target_flux[fit_mask], + 'comp_flux': ensemble_series['synthetic_reference_flux'][fit_mask], + 'target_flux_error': ( + np.sqrt(np.clip(np.abs(candidate_target_flux[fit_mask]), 1.0, None)) + if candidate_target_flux_error is None + else candidate_target_flux_error[fit_mask] + ), + 'comp_flux_error': ensemble_series['synthetic_reference_flux_error'][fit_mask], + 'source_indices': np.flatnonzero(fit_mask), + } + fit_result = build_stellar_variability_only_lightcurve( + prepared, + p_dict, + filter_diagnostics=filter_diagnostics, + comp_index=None, + comp_label=f"ENSEMBLE ({len(ensemble_members)} stars)", + comp_position=[member.get('position') for member in ensemble_members], + method_label=method_label, + plot_time_range=plot_time_range, + ) + if fit_result is not None: + annotate_stellar_variability_ensemble_differential_photometry( + fit_result, + ensemble_series, + ) + selected_source_indices = np.asarray( + fit_result.stellar_variability_source_indices, + dtype=int, + ) + fit_result.stellar_variability_ensemble_members = ensemble_members + fit_result.stellar_variability_ensemble_magnitudes = ( + ensemble_series['magnitude'][selected_source_indices] + ) + fit_result.stellar_variability_ensemble_magnitude_errors = ( + ensemble_series['magnitude_error'][selected_source_indices] + ) + fit_result.stellar_variability_ensemble_valid_member_counts = ( + ensemble_series['valid_member_count'][selected_source_indices] + ) + fit_result.stellar_variability_ensemble_calibration_error_clip = clip_summary + fit_result.stellar_variability_ensemble_selection = member_selection + fit_result.stellar_variability_target_catalog_profile = member_selection.get( + 'target_catalog_profile', {} + ) + scatter = getattr(fit_result, 'stellar_variability_scatter', np.nan) + exclusion_summary = getattr(fit_result, 'stellar_variability_transit_exclusion', {}) + ensemble_summary = { + 'comp_index': None, + 'key': 'ensemble', + 'label': f"Comparison ensemble ({len(ensemble_members)} comps)", + 'position': None, + 'aggregate_score': comparison_calibration.get('field_score', np.inf), + 'coverage_count': int(np.count_nonzero(fit_mask)), + 'coverage_total_frame_count': int(frame_count), + 'coverage_reference_count': np.nan, + 'coverage_min_required_count': LIGHTCURVE_MIN_VALID_POINTS, + 'coverage_rejected': False, + 'ensemble_frame_rejected_count': int(np.count_nonzero(~fit_mask)), + 'ensemble_frame_required_valid_pairs': len(ensemble_members), + 'ensemble_member_keys': [member['key'] for member in ensemble_members], + } + selected_result = { + 'rank': 0, + 'field_rank': 0, + 'comp_index': None, + 'ckey': 'ensemble', + 'label': ensemble_summary['label'], + 'position': None, + 'aggregate_score': ensemble_summary['aggregate_score'], + 'coverage_count': ensemble_summary['coverage_count'], + 'coverage_total_frame_count': frame_count, + 'coverage_reference_count': np.nan, + 'coverage_min_required_count': LIGHTCURVE_MIN_VALID_POINTS, + 'coverage_rejected': False, + 'ensemble_frame_rejected_count': ensemble_summary['ensemble_frame_rejected_count'], + 'ensemble_frame_required_valid_pairs': len(ensemble_members), + 'ensemble_member_keys': ensemble_summary['ensemble_member_keys'], + 'fit': fit_result, + 'full_reduction_fit': fit_result, + 'good_times': np.asarray(fit_result.time, dtype=float), + 'good_flux': np.asarray(fit_result.detrended, dtype=float), + 'good_unc': np.asarray(fit_result.detrendederr, dtype=float), + 'good_airmass': np.asarray(fit_result.airmass, dtype=float), + 'good_jd_times': np.asarray(fit_result.jd_times, dtype=float), + 'good_exposure_times_seconds': getattr( + fit_result, + 'stellar_variability_exposure_times_seconds', + None, + ), + 'good_target_flux_error': fit_result.stellar_variability_target_flux_error, + 'good_comp_flux_error': fit_result.stellar_variability_comp_flux_error, + 'tflux_fit': fit_result.stellar_variability_target_flux, + 'cflux_fit': fit_result.stellar_variability_comp_flux, + 'tflux_fit_error': fit_result.stellar_variability_target_flux_error, + 'cflux_fit_error': fit_result.stellar_variability_comp_flux_error, + 'source_indices': selected_source_indices, + 'duration_samples': np.asarray( + [exclusion_summary.get('duration_days', np.nan)], + dtype=float, + ), + 'data_highres': np.ones(1000, dtype=float), + 'fit_diagnostics': fit_diagnostics, + 'eebls_snr': np.nan, + 'transit_delta_bic': np.nan, + 'residual_scatter': scatter, + 'target_model_scatter_basis': 'out-of-transit comparison ensemble scatter', + 'projected_full_residual_scatter': scatter, + 'selection_scatter': scatter, + 'selection_scatter_basis': 'out-of-transit comparison ensemble scatter', + 'target_comp_scatter': target_comp_flux_scatter( + fit_result.stellar_variability_target_flux, + fit_result.stellar_variability_comp_flux, + ), + 'ktmf_metric': np.nan, + 'ktmf_contributions': [], + 'fit_point_count': int(np.asarray(fit_result.time).size), + 'failure_reason': fit_diagnostics.get('failure_reason'), + 'parameter_summary': None, + 'transit_qc_status': 'SKIPPED', + 'transit_qc_summary': 'Stellar variability only mode skipped transit fitting.', + 'rejected_by_transit_qc': False, + 'selected': True, + 'selection_reason': ( + 'selected: exact supplied comparison ensemble without star vetting' + if use_exactly_the_comps_provided + else ( + 'selected: fixed differential comparison ensemble' + if fixed_reference_mode + else 'selected: default calibrated ensemble of bright, unsaturated, ' + 'VSX-vetted comparison stars after high-side catalog-error clipping' + ) + ), + 'full_reduction_applied': True, + 'full_reduction_note': ( + 'completed the stellar-variability-only comparison ensemble reduction ' + 'without fitting a transit model.' + ), + 'stellar_variability_transit_exclusion': exclusion_summary, + 'reuse_selected_full_reduction_fit': True, + } + attempts.append(selected_result) + return { + 'ranked_summaries': ranked_summaries, + 'attempts': attempts, + 'selected_result': selected_result, + 'selection_metric': ( + 'exact_stellar_variability_ensemble' + if use_exactly_the_comps_provided + else 'stellar_variability_ensemble' + ), + 'stopped_after_first_qc_pass': False, + 'stopped_after_promising_partial': False, + } + if use_exactly_the_comps_provided: + log_info( + "Error: the exact supplied stellar-variability ensemble did not yield a usable " + "light curve; exact-comparison mode will not drop a member or select another star.", + error=True, + ) + return { + 'ranked_summaries': ranked_summaries, + 'attempts': attempts, + 'selected_result': None, + 'selection_metric': 'exact_stellar_variability_ensemble', + } + log_info( + "Warning: the stellar-variability comparison ensemble did not yield a usable " + "out-of-transit light curve; falling back to single-comparison selection.", + warn=True, + ) + else: + if use_exactly_the_comps_provided: + log_info( + "Error: the exact supplied comparison ensemble contained fewer than two usable " + "members; exact-comparison mode will not fall back to one star.", + error=True, + ) + return { + 'ranked_summaries': ranked_summaries, + 'attempts': attempts, + 'selected_result': None, + 'selection_metric': 'exact_stellar_variability_ensemble', + } + log_info( + "Warning: fewer than two bright, unsaturated, VSX-vetted comparison stars had usable " + "catalog calibrations after calibration-error clipping; falling back to single-comparison " + "selection.", + warn=True, + ) - ra_file, dec_file = get_ra_dec(wcs_header) - tar_radec = (ra_file[int(exotic_UIprevTPY)][int(exotic_UIprevTPX)], - dec_file[int(exotic_UIprevTPY)][int(exotic_UIprevTPX)]) + for field_rank, comp_summary in enumerate(ranked_summaries): + comp_index = comp_summary['comp_index'] + ckey = comp_summary.get('key', f"comp{comp_index + 1}") + comp_quality_mask = np.asarray( + comp_summary.get( + 'psf_quality_keep_mask', + psf_quality_mask_for_key( + psf_data, + ckey, + frame_count, + psf_flux_data=psf_flux_data if method == 'psf' else None, + ), + ), + dtype=bool, + ) + if comp_quality_mask.shape[0] != frame_count: + comp_quality_mask = psf_quality_mask_for_key( + psf_data, + ckey, + frame_count, + psf_flux_data=psf_flux_data if method == 'psf' else None, + ) - ra = ra_file[int(comp_star[1])][int(comp_star[0])] - dec = dec_file[int(comp_star[1])][int(comp_star[0])] + if method == 'psf': + target_shape_mask = target_psf_shape_quality_mask( + target_psf_quality_rows(psf_data, psf_flux_data=psf_flux_data), + psf_quality_rows_for_key(psf_data, ckey, psf_flux_data=psf_flux_data), + ) + if target_shape_mask.shape[0] != frame_count: + target_shape_mask = np.ones(frame_count, dtype=bool) + candidate_target_flux = mask_series_with_quality(target_flux, target_shape_mask) + candidate_target_flux_error = ( + None if target_flux_error is None else mask_series_with_quality(target_flux_error, target_shape_mask) + ) + comp_flux = psf_flux_series_from_rows(psf_flux_data[ckey], comp_quality_mask) + comp_flux_error = ( + mask_series_with_quality(psf_noise_data[ckey], comp_quality_mask) + if isinstance(psf_noise_data, dict) and ckey in psf_noise_data + else None + ) + else: + target_shape_mask = np.ones(frame_count, dtype=bool) + candidate_target_flux = target_flux + candidate_target_flux_error = target_flux_error + comp_flux = mask_series_with_quality(aper_data[ckey][:, aperture_index, annulus_index], comp_quality_mask) + comp_flux_error = ( + mask_series_with_quality( + aper_data[f"{ckey}_unc"][:, aperture_index, annulus_index], + comp_quality_mask, + ) + if f"{ckey}_unc" in aper_data + else None + ) - comp_radec.append((ra, dec)) + candidate_frame_keep_mask = np.asarray( + comp_summary.get('ensemble_frame_keep_mask', np.ones(times.shape[0], dtype=bool)), + dtype=bool, + ) + if candidate_frame_keep_mask.shape != times.shape: + candidate_frame_keep_mask = np.ones(times.shape[0], dtype=bool) + candidate_frame_clip_diagnostic = None + candidate_frame_diagnostic_keep_mask = candidate_frame_keep_mask | ~field_image_keep_mask + if np.any(~candidate_frame_diagnostic_keep_mask): + required_pairs = comp_summary.get( + 'ensemble_frame_required_valid_pairs', + COMPARISON_CANDIDATE_FRAME_OUTLIER_MIN_VALID_PAIRS, + ) + sigma_threshold = comp_summary.get('ensemble_frame_sigma', COMPARISON_IMAGE_OUTLIER_SIGMA) + candidate_frame_clip_diagnostic = build_time_rejection_diagnostic( + "Comparison-candidate intercomparison clip", + times, + candidate_frame_diagnostic_keep_mask, + note=( + "Dropped frames where this comparison star disagreed with the peer-star " + "intercomparison reference " + f"before target fitting; same-direction pairwise majority exceeded {sigma_threshold:.2f} sigma " + f"(min confirming pair count={required_pairs})." + ), + ) - first_image = fits.getdata(inputfiles[0]) - targ_sig_xy = fit_centroid(first_image, [exotic_UIprevTPX, exotic_UIprevTPY], 0)[3:5] + fit_mask = field_image_keep_mask & candidate_frame_keep_mask & comp_quality_mask & target_shape_mask + if method == 'psf': + fit_mask &= robust_target_reference_flux_mask(candidate_target_flux, comp_flux) + else: + fit_mask &= valid_comparison_frame_mask(candidate_target_flux) & valid_comparison_frame_mask(comp_flux) + + fit_diagnostics = diagnose_lightcurve_fit_inputs( + times[fit_mask], + candidate_target_flux[fit_mask], + comp_flux[fit_mask], + airmass[fit_mask], + target_flux_error=None if candidate_target_flux_error is None else candidate_target_flux_error[fit_mask], + comp_flux_error=None if comp_flux_error is None else comp_flux_error[fit_mask], + enforce_relative_flux_max=False, + expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), + ) + external_filter_diagnostics = [] + if field_image_clip_diagnostic is not None: + external_filter_diagnostics.append(field_image_clip_diagnostic) + if candidate_frame_clip_diagnostic is not None: + external_filter_diagnostics.append(candidate_frame_clip_diagnostic) + + fit_result = None + prepared = None + if fit_diagnostics.get('failure_reason') is None: + fit_result, prepared = build_stellar_variability_only_lightcurve_from_fluxes( + times[fit_mask], + candidate_target_flux[fit_mask], + comp_flux[fit_mask], + airmass[fit_mask], + p_dict, + jd_times=jd_times[fit_mask], + adaptive_summary=adaptive_summary, + target_flux_error=( + None if candidate_target_flux_error is None else candidate_target_flux_error[fit_mask] + ), + comp_flux_error=None if comp_flux_error is None else comp_flux_error[fit_mask], + exposure_times_seconds=( + None if exposure_times_array is None else exposure_times_array[fit_mask] + ), + gain_e_per_adu=gain_e_per_adu, + filter_diagnostics=external_filter_diagnostics, + comp_index=comp_index, + comp_label=comp_summary.get('label', f"Comp {comp_index + 1}"), + comp_position=comp_summary.get('position'), + method_label=method_label, + plot_time_range=plot_time_range, + ) + if fit_result is not None: + original_indices = np.flatnonzero(fit_mask) + source_indices = np.asarray(fit_result.stellar_variability_source_indices, dtype=int) + if source_indices.size and np.max(source_indices) < original_indices.size: + fit_result.stellar_variability_source_indices = original_indices[source_indices] + else: + fit_result.stellar_variability_source_indices = np.array([], dtype=int) + elif prepared is not None and prepared.get('applied'): + fit_diagnostics = dict(fit_diagnostics) + exclusion = stellar_variability_out_of_transit_mask(prepared.get('time'), p_dict)[1] + fit_diagnostics.update({ + 'failed_stage': 'stellar_variability_transit_window', + 'failure_reason': ( + "too few out-of-transit points remained after excluding the predicted " + "start-ingress to end-egress transit window." + ), + 'transit_window_rejected_point_count': exclusion.get('rejected_point_count', 0), + }) + else: + fit_diagnostics = ensure_lightcurve_fit_failure_reason( + fit_diagnostics, + fit_result, + failed_stage='stellar_variability_photometry', + failure_reason=( + "the raw comparison-candidate photometry did not yield a usable " + "out-of-transit stellar-variability light curve." + ), + ) + + source_indices = np.asarray( + getattr(fit_result, 'stellar_variability_source_indices', np.array([], dtype=int)), + dtype=int, + ) + tflux_fit = getattr(fit_result, 'stellar_variability_target_flux', np.array([], dtype=float)) + cflux_fit = getattr(fit_result, 'stellar_variability_comp_flux', np.array([], dtype=float)) + tflux_fit_error = getattr(fit_result, 'stellar_variability_target_flux_error', np.array([], dtype=float)) + cflux_fit_error = getattr(fit_result, 'stellar_variability_comp_flux_error', np.array([], dtype=float)) + scatter = getattr(fit_result, 'stellar_variability_scatter', np.nan) + exclusion_summary = getattr(fit_result, 'stellar_variability_transit_exclusion', {}) + + attempt = { + 'rank': field_rank, + 'field_rank': field_rank, + 'comp_index': comp_index, + 'ckey': ckey, + 'label': comp_summary.get('label', f"Comp {comp_index + 1}"), + 'position': comp_summary.get('position'), + 'aggregate_score': comp_summary.get('aggregate_score', np.inf), + 'coverage_count': comp_summary.get('coverage_count', 0), + 'coverage_total_frame_count': comp_summary.get('coverage_total_frame_count', 0), + 'coverage_reference_count': comp_summary.get('coverage_reference_count', np.nan), + 'coverage_min_required_count': comp_summary.get('coverage_min_required_count', 0), + 'coverage_rejected': comp_summary.get('coverage_rejected', False), + 'ensemble_frame_rejected_count': comp_summary.get('ensemble_frame_rejected_count', 0), + 'ensemble_frame_required_valid_pairs': comp_summary.get('ensemble_frame_required_valid_pairs', 0), + 'fit': fit_result, + 'full_reduction_fit': fit_result, + 'good_times': np.asarray(getattr(fit_result, 'time', np.array([], dtype=float)), dtype=float), + 'good_flux': np.asarray(getattr(fit_result, 'detrended', np.array([], dtype=float)), dtype=float), + 'good_unc': np.asarray(getattr(fit_result, 'detrendederr', np.array([], dtype=float)), dtype=float), + 'good_airmass': np.asarray(getattr(fit_result, 'airmass', np.array([], dtype=float)), dtype=float), + 'good_jd_times': np.asarray(getattr(fit_result, 'jd_times', np.array([], dtype=float)), dtype=float), + 'good_exposure_times_seconds': getattr( + fit_result, + 'stellar_variability_exposure_times_seconds', + None, + ), + 'good_target_flux_error': tflux_fit_error, + 'good_comp_flux_error': cflux_fit_error, + 'tflux_fit': tflux_fit, + 'cflux_fit': cflux_fit, + 'tflux_fit_error': tflux_fit_error, + 'cflux_fit_error': cflux_fit_error, + 'source_indices': source_indices, + 'duration_samples': np.array( + [exclusion_summary.get('duration_days', np.nan)], + dtype=float, + ), + 'data_highres': np.ones(1000, dtype=float), + 'fit_diagnostics': fit_diagnostics, + 'eebls_snr': np.nan, + 'transit_delta_bic': np.nan, + 'residual_scatter': scatter, + 'target_model_scatter_basis': 'out-of-transit normalized target/reference scatter', + 'projected_full_residual_scatter': scatter, + 'selection_scatter': scatter, + 'selection_scatter_basis': 'out-of-transit normalized target/reference scatter', + 'target_comp_scatter': target_comp_flux_scatter(tflux_fit, cflux_fit), + 'ktmf_metric': np.nan, + 'ktmf_contributions': [], + 'fit_point_count': int(np.asarray(tflux_fit).size), + 'failure_reason': fit_diagnostics.get('failure_reason'), + 'parameter_summary': None, + 'transit_qc_status': 'SKIPPED', + 'transit_qc_summary': 'Stellar variability only mode skipped transit fitting.', + 'rejected_by_transit_qc': False, + 'selected': False, + 'selection_reason': None, + 'full_reduction_applied': fit_result is not None, + 'full_reduction_note': ( + "completed the stellar-variability-only reduction without fitting a transit model." + if fit_result is not None else None + ), + 'stellar_variability_transit_exclusion': exclusion_summary, + 'reuse_selected_full_reduction_fit': fit_result is not None, + } + attempts.append(attempt) + scatter_text = format_residual_scatter(scatter) + if fit_result is None: + log_info( + f" {attempt['label']}: no usable stellar-variability-only light curve " + f"({attempt['failure_reason']})." + ) + else: + rejected = exclusion_summary.get('rejected_point_count', 0) + log_info( + f" {attempt['label']}: out-of-transit scatter={scatter_text}; " + f"used {attempt['fit_point_count']} point(s), excluded {rejected} predicted in-transit point(s)." + ) - # aperture size in stdev (sigma) of PSF - aper = 3 * max(targ_sig_xy) - annulus = 10 + eligible_attempts = [ + attempt for attempt in attempts + if attempt.get('fit') is not None and np.isfinite(attempt.get('selection_scatter', np.nan)) + ] + selected_result = None + if eligible_attempts: + selected_result = min( + eligible_attempts, + key=lambda attempt: ( + attempt.get('selection_scatter', np.inf), + attempt.get('aggregate_score', np.inf), + attempt.get('field_rank', np.inf), + ), + ) + selected_result['selected'] = True + selected_result['selection_reason'] = ( + "selected: lowest out-of-transit normalized target/reference scatter among " + "comparison-star calibration candidates" + ) + selected_scatter = selected_result.get('selection_scatter', np.nan) + for attempt in attempts: + if attempt is selected_result: + continue + if attempt.get('fit') is None: + continue + attempt['selection_reason'] = ( + "not selected: out-of-transit normalized target/reference scatter " + f"{format_residual_scatter(attempt.get('selection_scatter', np.nan))} was higher than " + f"the selected {format_residual_scatter(selected_scatter)}" + ) - # alloc psf fitting param - psf_data = { - # x-cent, y-cent, amplitude, sigma-x, sigma-y, rotation, offset - 'target': np.zeros((len(inputfiles), 7)), # PSF fit - 'comp': np.zeros((len(inputfiles), 7)) - } - tar_comp_dist = { - 'comp': np.zeros(2, dtype=int) + return { + 'ranked_summaries': ranked_summaries, + 'attempts': attempts, + 'selected_result': selected_result, + 'selection_metric': 'stellar_variability_scatter', + 'stopped_after_first_qc_pass': False, + 'stopped_after_promising_partial': False, } - # open files, calibrate, align, photometry - for i, fileName in enumerate(inputfiles): - plateStatus.setCurrentFilename(fileName) - hdul = fits.open(name=fileName, memmap=False, cache=False, lazy_load_hdus=False, - ignore_missing_end=True) - - extension = 0 - image_header = hdul[extension].header - while image_header["NAXIS"] == 0: - extension += 1 - image_header = hdul[extension].header - # TIME - timeVal = img_time_bjd_tdb(image_header, p_dict, info_dict) - timeList.append(timeVal) +def fortuitous_ensemble_flux_maps( + comparison_calibration, + psf_data, + aper_data, + psf_flux_data=None, + psf_noise_data=None): + ranked_summaries = ranked_comparison_calibration_summaries(comparison_calibration) + method = comparison_calibration.get('method') + aperture_index = comparison_calibration.get('a') + annulus_index = comparison_calibration.get('an') + frame_count = np.asarray(comparison_calibration.get('field_image_keep_mask', [])).size + if frame_count == 0: + if method == 'psf': + frame_count = np.asarray(psf_data.get('target', [])).shape[0] + elif aper_data is not None: + frame_count = np.asarray(aper_data.get('target', [])).shape[0] + flux_map = {} + error_map = {} + psf_flux_source = psf_flux_data_source(psf_data, psf_flux_data) + for summary in ranked_summaries: + ckey = summary.get('key') + if not ckey: + continue + quality_mask = np.asarray( + summary.get( + 'psf_quality_keep_mask', + psf_quality_mask_for_key( + psf_data, + ckey, + frame_count, + psf_flux_data=psf_flux_source if method == 'psf' else None, + ), + ), + dtype=bool, + ) + if quality_mask.shape != (frame_count,): + quality_mask = np.ones(frame_count, dtype=bool) + if method == 'psf': + if ckey not in psf_flux_source: + continue + flux_map[ckey] = psf_flux_series_from_rows(psf_flux_source[ckey], quality_mask) + if isinstance(psf_noise_data, dict) and ckey in psf_noise_data: + error_map[ckey] = mask_series_with_quality(psf_noise_data[ckey], quality_mask) + else: + if aper_data is None or ckey not in aper_data: + continue + flux_map[ckey] = mask_series_with_quality( + aper_data[ckey][:, aperture_index, annulus_index], + quality_mask, + ) + error_key = f"{ckey}_unc" + if error_key in aper_data: + error_map[ckey] = mask_series_with_quality( + aper_data[error_key][:, aperture_index, annulus_index], + quality_mask, + ) + return ranked_summaries, flux_map, error_map + + +def fortuitous_variable_target_series( + variable, + comparison_calibration, + psf_data, + aper_data, + psf_flux_data=None, + psf_noise_data=None, + comp_overexposed_masks=None): + ckey = variable.get('tracking_key') + method = comparison_calibration.get('method') + frame_count = np.asarray(comparison_calibration.get('field_image_keep_mask', [])).size + psf_flux_source = psf_flux_data_source(psf_data, psf_flux_data) + quality_mask = psf_quality_mask_for_key( + psf_data, + ckey, + frame_count, + psf_flux_data=psf_flux_source if method == 'psf' else None, + ) + quality_mask = np.asarray(quality_mask, dtype=bool) + if quality_mask.shape != (frame_count,): + quality_mask = np.ones(frame_count, dtype=bool) + if isinstance(comp_overexposed_masks, dict) and ckey in comp_overexposed_masks: + overexposed = np.asarray(comp_overexposed_masks[ckey], dtype=bool) + if overexposed.shape == quality_mask.shape: + quality_mask &= ~overexposed + + if method == 'psf': + if ckey not in psf_flux_source: + return None, None, quality_mask + flux = psf_flux_series_from_rows(psf_flux_source[ckey], quality_mask) + error = None + if isinstance(psf_noise_data, dict) and ckey in psf_noise_data: + error = mask_series_with_quality(psf_noise_data[ckey], quality_mask) + return flux, error, quality_mask + + aperture_index = comparison_calibration.get('a') + annulus_index = comparison_calibration.get('an') + if aper_data is None or ckey not in aper_data: + return None, None, quality_mask + flux = mask_series_with_quality( + aper_data[ckey][:, aperture_index, annulus_index], + quality_mask, + ) + error_key = f"{ckey}_unc" + error = None + if error_key in aper_data: + error = mask_series_with_quality( + aper_data[error_key][:, aperture_index, annulus_index], + quality_mask, + ) + return flux, error, quality_mask - # IMAGES - imageData = hdul[extension].data - if i == 0: - firstImage = np.copy(imageData) +def fortuitous_variable_target_metadata(variable): + reference_mode = variable.get('reference_mode', 'ensemble') + reference_description = ( + 'single-comparison calibrated' + if reference_mode == 'single_comparison' + else 'ensemble-calibrated' + ) + output_error_limit = _finite_float( + variable.get('output_magnitude_error_limit'), + FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR, + ) + return { + 'name': variable.get('name'), + 'auid': variable.get('auid'), + 'variable_type': variable.get('variable_type'), + 'ra_deg': variable.get('ra'), + 'dec_deg': variable.get('dec'), + 'pixel_position': variable.get('pos'), + 'vsx_period_days': variable.get('period_days'), + 'vsx_amplitude_mag': variable.get('amplitude_mag'), + 'classification_folder': variable.get('category'), + 'classification_rule': ( + 'optimal_variables requires VSX period <= 10 days and amplitude >= 0.3 mag; ' + 'all other retained VSX stars use normal.' + ), + 'reference_aperture_flux_adu': variable.get('aperture_flux_adu'), + 'reference_count_rate_adu_per_second': variable.get('count_rate_adu_per_second'), + 'reference_estimated_magnitude_error': variable.get('estimated_magnitude_error'), + 'reference_sky_background_adu_per_pixel': variable.get( + 'reference_sky_background_adu_per_pixel' + ), + 'reference_sky_sigma_adu': variable.get('reference_sky_sigma_adu'), + 'reference_aperture_pixels': variable.get('reference_aperture_pixels'), + 'reference_sky_pixels': variable.get('reference_sky_pixels'), + 'reference_flux_error_adu': variable.get('reference_flux_error_adu'), + 'reference_noise_components_adu': variable.get('reference_noise_components_adu'), + 'detection_magnitude_error_limit': FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR, + 'output_magnitude_error_limit': output_error_limit, + 'output_magnitude_error_rule': ( + f'Only frames with a finite positive {reference_description} magnitude error below ' + f'{output_error_limit:.2f} mag are written.' + ), + 'reference_mode': reference_mode, + 'comparison_label': variable.get('comparison_label'), + 'saturation_rejection_scope': ( + 'Frames are rejected using this VSX target own overexposure mask; ' + 'the exoplanet target overexposure mask is not applied.' + ), + 'input_frame_count': variable.get('input_frame_count'), + 'target_overexposure_rejected_frame_count': variable.get( + 'target_overexposure_rejected_frame_count' + ), + 'target_quality_rejected_frame_count': variable.get( + 'target_quality_rejected_frame_count' + ), + 'output_magnitude_error_rejected_frame_count': variable.get( + 'output_magnitude_error_rejected_frame_count' + ), + 'output_magnitude_error_qualified_frame_count': variable.get( + 'output_magnitude_error_qualified_frame_count' + ), + 'output_magnitude_error_min': variable.get('output_magnitude_error_min'), + 'output_magnitude_error_median': variable.get('output_magnitude_error_median'), + 'output_magnitude_error_max': variable.get('output_magnitude_error_max'), + 'valid_output_frame_count': variable.get('valid_output_frame_count'), + } - sys.stdout.write(f"Finding transformation {i + 1} of {len(inputfiles)} : {fileName}\n") - log.debug(f"Finding transformation {i + 1} of {len(inputfiles)} : {fileName}\n") - sys.stdout.flush() +def fortuitous_output_magnitude_error_limit(members): + for member in members or []: + star = member.get('star', {}) if isinstance(member, dict) else {} + if ( + bool(star.get('uses_relaxed_bv_error_limit', False)) + and str(star.get('mag_band') or '').strip().upper() in {'B', 'V'} + ): + return CATALOG_BV_REFERENCE_MAGNITUDE_ERROR_FALLBACK_MAX + return FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR + + +def clear_previous_fortuitous_variable_products(variable_dir): + output_dir = Path(variable_dir) + for prefix in ( + 'AID_AAVSO_', + 'StellarVariability_', + 'DifferentialMagnitude_', + 'EnsembleSelection_', + 'FortuitousVariableStatus_', + ): + for path in output_dir.glob(f'{prefix}*'): + if path.is_file(): + path.unlink() + aavso_dir = output_dir / AAVSO_OUTPUT_FOLDER_NAME + if aavso_dir.is_dir(): + for path in aavso_dir.glob('AID_AAVSO_*'): + if path.is_file(): + path.unlink() try: - wcs_hdr = search_wcs(fileName) - if not wcs_hdr.is_celestial: - raise Exception - - if i == 0: - tx, ty = exotic_UIprevTPX, exotic_UIprevTPY - else: - pix_coords = wcs_hdr.world_to_pixel_values(tar_radec[0], tar_radec[1]) - tx, ty = pix_coords[0].take(0), pix_coords[1].take(0) + aavso_dir.rmdir() + except OSError: + pass + plot_path = output_dir / 'working_artifacts' / 'Stellar_Variability.png' + if plot_path.is_file(): + plot_path.unlink() + for differential_plot in ( + output_dir / 'Stellar_Variability_DifferentialMagnitude.png', + output_dir / 'working_artifacts' / 'Stellar_Variability_DifferentialMagnitude.png', + ): + if differential_plot.is_file(): + differential_plot.unlink() + working_artifacts_dir = output_dir / 'working_artifacts' + if working_artifacts_dir.is_dir(): + try: + working_artifacts_dir.rmdir() + except OSError: + pass + if output_dir.is_dir(): + try: + output_dir.rmdir() + except OSError: + pass - psf_data['target'][i] = fit_centroid(imageData, [tx, ty], 0) - if i != 0 and np.abs((psf_data['target'][i][2] - psf_data['target'][i - 1][2]) - / psf_data['target'][i - 1][2]) > 0.5: - raise Exception +def process_fortuitous_variables( + variables, + comparison_calibration, + calibration_stars, + times, + jd_times, + airmass, + psf_data, + aper_data, + info_dict, + psf_flux_data=None, + psf_noise_data=None, + comp_overexposed_masks=None, + exposure_times_seconds=None, + observed_filter=None, + use_single_comparison=USE_SINGLE_COMPARISON_FOR_FORTUITOUS_VARIABLES_DEFAULT, + maximum_number_of_ensemble_comparisons_for_stellar_variability= + STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS): + if not variables or comparison_calibration is None: + return [] + times = np.asarray(times, dtype=float) + jd_times = np.asarray(jd_times, dtype=float) + airmass = np.asarray(airmass, dtype=float) + if not (times.shape == jd_times.shape == airmass.shape): + return [] + use_single_comparison = bool(use_single_comparison) + required_comparison_members = ( + 1 if use_single_comparison else STELLAR_VARIABILITY_ENSEMBLE_MIN_MEMBERS + ) + reference_mode = 'single_comparison' if use_single_comparison else 'ensemble' + + ranked_summaries, comp_flux_map, comp_error_map = fortuitous_ensemble_flux_maps( + comparison_calibration, + psf_data, + aper_data, + psf_flux_data=psf_flux_data, + psf_noise_data=psf_noise_data, + ) + if len(ranked_summaries) < required_comparison_members: + log_info( + "Warning: fortuitous-variable photometry skipped because fewer than " + f"{required_comparison_members} independent non-variable comparison candidate(s) " + "were usable.", + warn=True, + ) + return [] - pix_coords = wcs_hdr.world_to_pixel_values(comp_radec[0][0], comp_radec[0][1]) - cx, cy = pix_coords[0].take(0), pix_coords[1].take(0) - psf_data['comp'][i] = fit_centroid(imageData, [cx, cy], 1) + field_keep_mask = np.asarray( + comparison_calibration.get('field_image_keep_mask', np.ones(times.shape, dtype=bool)), + dtype=bool, + ) + if field_keep_mask.shape != times.shape: + field_keep_mask = np.ones(times.shape, dtype=bool) + exposure_array = None + if exposure_times_seconds is not None: + exposure_array = np.asarray(exposure_times_seconds, dtype=float) + if exposure_array.shape != times.shape: + exposure_array = None + + base_dir = Path(info_dict['save']) / 'variables' + base_dir.mkdir(parents=True, exist_ok=True) + for combined_aavso_dir in (base_dir, base_dir / AAVSO_OUTPUT_FOLDER_NAME): + for stale_combined_aid in combined_aavso_dir.glob( + 'AID_AAVSO_FortuitousVariables_*.txt'): + if stale_combined_aid.is_file(): + stale_combined_aid.unlink() + results = [] + combined_vsp_params = [] + logged_comparison_gap_rejections = set() + for variable in variables: + variable = dict(variable) + variable['reference_mode'] = reference_mode + variable_name = variable.get('name') or 'VSX variable' + category = variable.get('category') or 'normal' + if category == 'rest_of_the_variables': + category = 'normal' + variable['category'] = category + variable['input_frame_count'] = int(times.size) + variable_overexposed_mask = np.zeros(times.shape, dtype=bool) + tracking_key = variable.get('tracking_key') + if isinstance(comp_overexposed_masks, dict) and tracking_key in comp_overexposed_masks: + candidate_mask = np.asarray(comp_overexposed_masks[tracking_key], dtype=bool) + if candidate_mask.shape == times.shape: + variable_overexposed_mask = candidate_mask + variable['target_overexposure_rejected_frame_count'] = int( + np.count_nonzero(variable_overexposed_mask) + ) + variable_dir = base_dir / category / safe_output_filename(variable_name, extension='') + clear_previous_fortuitous_variable_products(variable_dir) + try: + target_flux, target_flux_error, target_quality_mask = fortuitous_variable_target_series( + variable, + comparison_calibration, + psf_data, + aper_data, + psf_flux_data=psf_flux_data, + psf_noise_data=psf_noise_data, + comp_overexposed_masks=comp_overexposed_masks, + ) + if target_flux is None: + raise ValueError('no usable tracked flux series') + variable['target_quality_rejected_frame_count'] = int( + np.count_nonzero(~np.asarray(target_quality_mask, dtype=bool)) + ) - if i != 0: - if not (tar_comp_dist['comp'][0] - 1 <= abs(int(psf_data['comp'][0][0]) - int(psf_data['target'][i][0])) <= tar_comp_dist['comp'][0] + 1 and - tar_comp_dist['comp'][1] - 1 <= abs(int(psf_data['comp'][0][1]) - int(psf_data['target'][i][1])) <= tar_comp_dist['comp'][1] + 1) or \ - np.abs((psf_data['comp'][i][2] - psf_data['comp'][i - 1][2]) / psf_data['comp'][i - 1][2]) > 0.5: - raise Exception + member_selection = select_stellar_variability_ensemble_members( + ranked_summaries, + calibration_stars, + comp_flux_map, + observed_filter=observed_filter, + target_catalog_match=variable.get('catalog_match'), + max_members=( + 1 + if use_single_comparison + else maximum_number_of_ensemble_comparisons_for_stellar_variability + ), + min_members=required_comparison_members, + times=times, + ) + variable['comparison_gap_stability'] = member_selection.get('gap_stability', {}) + variable['comparison_gap_rejected_candidates'] = [ + candidate + for candidate in member_selection.get('rejected', []) + if candidate.get('maximum_absolute_step_magnitude') is not None + ] + for rejected_candidate in variable['comparison_gap_rejected_candidates']: + rejection_identity = ( + rejected_candidate.get('key'), + rejected_candidate.get('maximum_absolute_step_magnitude'), + ) + if rejection_identity in logged_comparison_gap_rejections: + continue + logged_comparison_gap_rejections.add(rejection_identity) + log_info( + "Fortuitous-variable comparison rejected across acquisition gap: " + f"{rejected_candidate.get('label') or rejected_candidate.get('key')} " + f"at {rejected_candidate.get('position')}, " + f"step={rejected_candidate.get('maximum_absolute_step_magnitude'):.4f} mag, " + f"significance={rejected_candidate.get('maximum_step_significance'):.2f} sigma." + ) + members = member_selection.get('members', []) + apparent_reference_available = len(members) >= required_comparison_members + if not apparent_reference_available: + # A catalogue magnitude is optional for differential + # photometry. Retain the normal vetted/ranked candidate order + # and build the requested single or ensemble reference from + # instrumental flux alone. + raw_members = [] + member_limit = ( + 1 + if use_single_comparison + else parse_maximum_number_of_ensemble_comparisons_for_stellar_variability( + maximum_number_of_ensemble_comparisons_for_stellar_variability + ) + ) + for summary in ranked_summaries: + key = summary.get('key') + flux = np.asarray(comp_flux_map.get(key, []), dtype=float) + if key not in comp_flux_map or np.count_nonzero( + np.isfinite(flux) & (flux > 0) + ) < LIGHTCURVE_MIN_VALID_POINTS: + continue + raw_members.append({ + 'key': key, + 'comp_index': summary.get('comp_index'), + 'label': summary.get('label', key), + 'position': summary.get('position'), + 'summary': summary, + }) + if len(raw_members) >= member_limit: + break + members = raw_members + if len(members) < required_comparison_members: + raise ValueError( + f'fewer than {required_comparison_members} usable differential ' + 'comparison member(s)' + ) + member_selection = { + **member_selection, + 'members': members, + 'fixed_reference': True, + 'apparent_calibration_available': False, + } + output_magnitude_error_limit = fortuitous_output_magnitude_error_limit(members) + variable['output_magnitude_error_limit'] = output_magnitude_error_limit + if apparent_reference_available: + ensemble_series = build_stellar_variability_calibrated_ensemble_series( + target_flux, + target_flux_error, + comp_flux_map, + comp_error_map, + members, + minimum_members=required_comparison_members, + validity_mask_func=( + robust_flux_floor_mask + if comparison_calibration.get('method') == 'psf' + else valid_comparison_frame_mask + ), + ) else: - tar_comp_dist['comp'][0] = abs(int(psf_data['comp'][0][0]) - int(psf_data['target'][0][0])) - tar_comp_dist['comp'][1] = abs(int(psf_data['comp'][0][1]) - int(psf_data['target'][0][1])) - except Exception: - if i == 0: - tform = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) + relative_series = build_relative_comparison_ensemble_series( + target_flux, + target_flux_error, + comp_flux_map, + comp_error_map, + [member.get('key') for member in members], + ) + relative_flux = np.asarray( + relative_series.get('relative_flux', np.full(target_flux.shape, np.nan)), + dtype=float, + ) + relative_flux_error = np.asarray( + relative_series.get( + 'relative_flux_error', + np.full(target_flux.shape, np.nan), + ), + dtype=float, + ) + with np.errstate(divide='ignore', invalid='ignore'): + instrumental_magnitude_error = ( + (2.5 / np.log(10.0)) + * np.abs(relative_flux_error / relative_flux) + ) + ensemble_series = { + 'applied': relative_series.get('applied', False), + 'failure_reason': relative_series.get('failure_reason'), + 'relative_flux': relative_flux, + 'relative_flux_error': relative_flux_error, + 'synthetic_reference_flux': relative_series.get('reference_flux'), + 'synthetic_reference_flux_error': relative_series.get( + 'reference_flux_error' + ), + 'raw_reference_flux': relative_series.get('reference_flux'), + 'raw_reference_flux_error': relative_series.get('reference_flux_error'), + 'magnitude': np.full(target_flux.shape, np.nan, dtype=float), + 'magnitude_error': instrumental_magnitude_error, + 'valid_member_count': np.where( + relative_series.get( + 'valid_mask', + np.zeros(target_flux.shape, dtype=bool), + ), + len(members), + 0, + ), + } + if not ensemble_series.get('applied'): + raise ValueError(ensemble_series.get('failure_reason') or 'ensemble combination failed') + + base_valid = ( + field_keep_mask + & target_quality_mask + & np.isfinite(ensemble_series['relative_flux']) + & (ensemble_series['relative_flux'] > 0) + & np.isfinite(ensemble_series['relative_flux_error']) + & (ensemble_series['relative_flux_error'] > 0) + ) + magnitude_errors = np.asarray(ensemble_series['magnitude_error'], dtype=float) + magnitude_error_valid = ( + np.isfinite(magnitude_errors) + & (magnitude_errors > 0) + & (magnitude_errors <= output_magnitude_error_limit) + ) + valid = base_valid & magnitude_error_valid + eligible_magnitude_errors = magnitude_errors[base_valid & np.isfinite(magnitude_errors)] + variable['output_magnitude_error_rejected_frame_count'] = int( + np.count_nonzero(base_valid & ~magnitude_error_valid) + ) + variable['output_magnitude_error_qualified_frame_count'] = int(np.count_nonzero(valid)) + if eligible_magnitude_errors.size: + variable['output_magnitude_error_min'] = float(np.nanmin(eligible_magnitude_errors)) + variable['output_magnitude_error_median'] = float(np.nanmedian(eligible_magnitude_errors)) + variable['output_magnitude_error_max'] = float(np.nanmax(eligible_magnitude_errors)) + if np.count_nonzero(valid) < LIGHTCURVE_MIN_VALID_POINTS: + raise ValueError( + f'fewer than five {reference_mode.replace("_", "-")}-calibrated frames ' + 'have internal magnitude error ' + f'at or below {output_magnitude_error_limit:.3f} mag' + ) + variable['valid_output_frame_count'] = int(np.count_nonzero(valid)) + target_error_values = target_flux_error + if target_error_values is None: + target_error_values = source_flux_uncertainty_from_counts(target_flux) + target_error_values = np.asarray(target_error_values, dtype=float) + if use_single_comparison: + selected_member = members[0] + selected_comparison_key = selected_member.get('key') + selected_comparison_flux = np.asarray( + comp_flux_map[selected_comparison_key], + dtype=float, + ) + selected_comparison_error = None + if selected_comparison_key in comp_error_map: + selected_comparison_error = np.asarray( + comp_error_map[selected_comparison_key], + dtype=float, + ) + if ( + selected_comparison_error is None + or selected_comparison_error.shape != selected_comparison_flux.shape + ): + selected_comparison_error = source_flux_uncertainty_from_counts( + selected_comparison_flux + ) + reference_label = selected_member.get('label') or selected_comparison_key + reference_position = selected_member.get('position') + variable['comparison_label'] = reference_label else: - tform = transformation(np.array([imageData, firstImage]), fileName) - - tx, ty = tform([exotic_UIprevTPX, exotic_UIprevTPY])[0] - psf_data['target'][i] = fit_centroid(imageData, [tx, ty], 0) - - cx, cy = tform(comp_star)[0] - psf_data['comp'][i] = fit_centroid(imageData, [cx, cy], 1) - - if i == 0: - tar_comp_dist['comp'][0] = abs(int(psf_data['comp'][0][0]) - int(psf_data['target'][0][0])) - tar_comp_dist['comp'][1] = abs(int(psf_data['comp'][0][1]) - int(psf_data['target'][0][1])) - - # aperture photometry - if i == 0: - sigma = float((psf_data['target'][0][3] + psf_data['target'][0][4]) * 0.5) - aper *= sigma - annulus *= sigma - - tFlux = aperPhot(imageData, 0, psf_data['target'][i, 0], psf_data['target'][i, 1], aper, annulus)[0] - cFlux = aperPhot(imageData, 1, psf_data['comp'][i, 0], psf_data['comp'][i, 1], aper, annulus)[0] - norm_flux.append(tFlux / cFlux) - - # close file + delete from memory - hdul.close() - del hdul - # Replaced each loop, so clean up - del imageData - - ax.clear() - ax.set_title(target_name) - ax.set_ylabel('Normalized Flux') - ax.set_xlabel('Time (JD)') - ax.plot(timeList, norm_flux, 'bo') + selected_member = None + selected_comparison_flux = ensemble_series['synthetic_reference_flux'] + selected_comparison_error = ensemble_series['synthetic_reference_flux_error'] + reference_label = f"ENSEMBLE ({len(members)} stars)" + reference_position = [member.get('position') for member in members] + prepared = { + 'applied': True, + 'failure_reason': None, + 'time': times[valid], + 'flux': ensemble_series['relative_flux'][valid], + 'unc': ensemble_series['relative_flux_error'][valid], + 'airmass': airmass[valid], + 'jd_time': jd_times[valid], + 'exposure_time_seconds': None if exposure_array is None else exposure_array[valid], + 'target_flux': np.asarray(target_flux, dtype=float)[valid], + 'comp_flux': selected_comparison_flux[valid], + 'target_flux_error': target_error_values[valid], + 'comp_flux_error': selected_comparison_error[valid], + 'source_indices': np.flatnonzero(valid), + } + variable_period = _finite_float(variable.get('period_days'), 1.0) + if variable_period is None or variable_period <= 0: + variable_period = 1.0 + variable_prior = { + 'pPer': variable_period, + 'pPerUnc': np.nan, + 'midT': float(times[valid][0]), + 'midTUnc': np.nan, + 'rprs': np.nan, + 'rprsUnc': np.nan, + 'aRs': np.nan, + 'aRsUnc': np.nan, + 'inc': np.nan, + 'incUnc': np.nan, + 'ecc': 0.0, + 'omega': 0.0, + } + fit = build_stellar_variability_only_lightcurve( + prepared, + variable_prior, + comp_index=None, + comp_label=reference_label, + comp_position=reference_position, + method_label=comparison_calibration.get('method_label'), + plot_time_range=times, + ) + if fit is None: + raise ValueError('stellar-variability light curve construction failed') + if not use_single_comparison: + annotate_stellar_variability_ensemble_differential_photometry( + fit, + ensemble_series, + ) + variable_dir.mkdir(parents=True, exist_ok=True) + differential_csv_path = write_differential_magnitude_csv( + fit, + variable_dir, + variable_name, + observation_date=info_dict.get('date'), + observed_filter=observed_filter, + ) + plot_differential_magnitude( + fit, + variable_name, + variable_dir, + info_dict.get('date'), + observed_filter=observed_filter, + ) + apparent_output_error = None + vsp_params = [] + csv_path = None + try: + if use_single_comparison: + vsp_params = build_stellar_variability_params_from_fit( + fit, + selected_member.get('star', {}), + selected_member.get('position'), + reference_label, + variable_dir, + variable_name, + observed_filter=observed_filter, + observation_date=info_dict.get('date'), + ) + else: + selected_indices = np.asarray(fit.stellar_variability_source_indices, dtype=int) + fit.stellar_variability_ensemble_members = members + fit.stellar_variability_ensemble_magnitudes = ensemble_series['magnitude'][selected_indices] + fit.stellar_variability_ensemble_magnitude_errors = ( + ensemble_series['magnitude_error'][selected_indices] + ) + fit.stellar_variability_ensemble_valid_member_counts = ( + ensemble_series['valid_member_count'][selected_indices] + ) + fit.stellar_variability_ensemble_calibration_error_clip = member_selection.get( + 'calibration_error_clip', {} + ) + fit.stellar_variability_ensemble_selection = member_selection + fit.stellar_variability_target_catalog_profile = member_selection.get( + 'target_catalog_profile', {} + ) + + target_metadata = fortuitous_variable_target_metadata(variable) + vsp_params = build_stellar_variability_ensemble_params_from_fit( + fit, + variable_dir, + variable_name, + observed_filter=observed_filter, + observation_date=info_dict.get('date'), + target_metadata=target_metadata, + ) + if vsp_params: + csv_path = save_stellar_variability_magnitude_csv( + vsp_params, + variable_dir, + variable_name, + observation_date=info_dict.get('date'), + ) + except Exception as exc: + apparent_output_error = str(exc) + vsp_params = [] + log_info( + f"Warning: apparent-magnitude output was unavailable for {variable_name} " + f"({exc}); differential-magnitude products were retained.", + warn=True, + ) + variable_info = dict(info_dict) + variable_info['save'] = str(variable_dir) + variable_planet = {'sName': variable_name, 'pName': variable_name} + if vsp_params: + try: + AIDOutputFiles( + fit, + variable_planet, + variable_info, + variable.get('auid'), + None, + vsp_params, + ).aavso() + aid_variable_name = variable.get('auid') or variable_name + combined_vsp_params.extend([ + {**vsp_param, '_aid_name': aid_variable_name} + for vsp_param in vsp_params + ]) + except Exception as exc: + apparent_output_error = str(exc) + log_info( + f"Warning: AID apparent-magnitude output failed for {variable_name} " + f"({exc}); differential-magnitude products were retained.", + warn=True, + ) + results.append({ + 'name': variable_name, + 'auid': variable.get('auid'), + 'category': category, + 'output_directory': str(variable_dir), + 'point_count': int(len(fit.time)), + 'apparent_magnitude_point_count': len(vsp_params), + 'reference_mode': reference_mode, + 'comparison_label': reference_label, + 'selected_comparison_gap_stability': ( + selected_member.get('gap_stability') if selected_member is not None else None + ), + 'comparison_gap_stability': variable.get('comparison_gap_stability'), + 'comparison_gap_rejected_candidates': variable.get( + 'comparison_gap_rejected_candidates', [] + ), + 'comparison_member_count': len(members), + 'ensemble_member_count': len(members), + 'input_frame_count': variable.get('input_frame_count'), + 'target_overexposure_rejected_frame_count': variable.get( + 'target_overexposure_rejected_frame_count' + ), + 'output_magnitude_error_rejected_frame_count': variable.get( + 'output_magnitude_error_rejected_frame_count' + ), + 'output_magnitude_error_qualified_frame_count': variable.get( + 'output_magnitude_error_qualified_frame_count' + ), + 'output_magnitude_error_limit': variable.get( + 'output_magnitude_error_limit', + FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR, + ), + 'output_magnitude_error_max': variable.get('output_magnitude_error_max'), + 'magnitude_csv': str(csv_path) if csv_path else None, + 'differential_magnitude_csv': ( + str(differential_csv_path) if differential_csv_path else None + ), + 'apparent_magnitude_error': apparent_output_error, + 'status': 'completed', + }) + log_info( + f"Fortuitous-variable photometry completed for {variable_name}: " + f"{len(fit.time)} differential point(s), {len(vsp_params)} apparent point(s), " + f"reference={reference_label} " + f"({reference_mode}), outputs={variable_dir}." + ) + except Exception as exc: + clear_previous_fortuitous_variable_products(variable_dir) + results.append({ + 'name': variable_name, + 'auid': variable.get('auid'), + 'category': category, + 'output_directory': None, + 'reference_mode': reference_mode, + 'comparison_label': variable.get('comparison_label'), + 'comparison_gap_stability': variable.get('comparison_gap_stability'), + 'comparison_gap_rejected_candidates': variable.get( + 'comparison_gap_rejected_candidates', [] + ), + 'input_frame_count': variable.get('input_frame_count'), + 'target_overexposure_rejected_frame_count': variable.get( + 'target_overexposure_rejected_frame_count' + ), + 'output_magnitude_error_rejected_frame_count': variable.get( + 'output_magnitude_error_rejected_frame_count' + ), + 'output_magnitude_error_qualified_frame_count': variable.get( + 'output_magnitude_error_qualified_frame_count' + ), + 'output_magnitude_error_limit': variable.get( + 'output_magnitude_error_limit', + FORTUITOUS_VARIABLE_MAX_ESTIMATED_MAGNITUDE_ERROR, + ), + 'status': 'skipped', + 'reason': str(exc), + }) + log_info( + f"Warning: fortuitous-variable photometry skipped {variable_name} ({exc}).", + warn=True, + ) -def fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times=None): - # remove outliers - si = np.argsort(times) - dt = np.mean(np.diff(np.sort(times))) - ndt = int(25. / 24. / 60. / dt) * 2 + 1 - if ndt > len(times): - ndt = int(len(times)/4) * 2 + 1 - filtered_data = sigma_clip((tFlux / cFlux)[si], sigma=3, dt=max(5,ndt)) - arrayFinalFlux = (tFlux / cFlux)[si][~filtered_data] - f1 = tFlux[si][~filtered_data] - sigf1 = f1 ** 0.5 - f2 = cFlux[si][~filtered_data] - sigf2 = f2 ** 0.5 - if np.sum(cFlux) == len(cFlux): - arrayNormUnc = sigf1 - else: - arrayNormUnc = np.sqrt((sigf1 / f2) ** 2 + (sigf2 * f1 / f2 ** 2) ** 2) - arrayTimes = times[si][~filtered_data] - arrayJDTimes = jd_times[si][~filtered_data] - arrayAirmass = airmass[si][~filtered_data] - - # remove nans - nanmask = np.isnan(arrayFinalFlux) | np.isnan(arrayNormUnc) | np.isnan(arrayTimes) | np.isnan( - arrayAirmass) | np.less_equal(arrayFinalFlux, 0) | np.less_equal(arrayNormUnc, 0) - nanmask = nanmask | np.isinf(arrayFinalFlux) | np.isinf(arrayNormUnc) | np.isinf(arrayTimes) | np.isinf( - arrayAirmass) + base_dir.mkdir(parents=True, exist_ok=True) + combined_aid_path = None + combined_aid_error = None + if combined_vsp_params: + try: + combined_info = dict(info_dict) + combined_info['save'] = str(base_dir) + combined_target = { + 'sName': 'FortuitousVariables', + 'pName': 'FortuitousVariables', + } + combined_aid_path = AIDOutputFiles( + None, + combined_target, + combined_info, + None, + None, + combined_vsp_params, + ).combined_aavso() + log_info( + f"Combined fortuitous-variable AID file written with " + f"{len(combined_vsp_params)} row(s): {combined_aid_path}." + ) + except Exception as exc: + combined_aid_error = str(exc) + log_info( + f"Warning: could not create the combined fortuitous-variable AID file ({exc}).", + warn=True, + ) + manifest_path = base_dir / safe_output_filename( + 'FortuitousVariables', + filename_date_token(info_dict.get('date')), + extension='json', + ) + with manifest_path.open('w', encoding='utf-8') as handle: + manifest_payload = { + 'variables': results, + 'combined_aid': str(combined_aid_path) if combined_aid_path else None, + } + if combined_aid_error: + manifest_payload['combined_aid_error'] = combined_aid_error + json.dump(stellar_variability_json_safe(manifest_payload), handle, indent=2, sort_keys=True) + handle.write('\n') + return results + + +def limited_ensemble_comparison_keys( + ranked_summaries, + maximum_number_of_ensemble_comparisons_for_transit): + ensemble_limit = parse_maximum_number_of_ensemble_comparisons_for_transit( + maximum_number_of_ensemble_comparisons_for_transit + ) + keys = [ + summary.get('key') + for summary in ranked_summaries or [] + if summary.get('key') + ] + return keys[:ensemble_limit] + + +def fit_ranked_comparison_calibration_candidates(times, jd_times, airmass, ld, p_dict, comparison_calibration, + psf_data, aper_data, target_psf_flux, + psf_flux_data=None, + psf_noise_data=None, + plot_time_range=None, + disable_vertical_flux_normalization=False, + detrend_on_outoftransit_baseline=True, + use_impactparameter_rather_than_inclination_to_fit=True, + use_eebls_to_initialize_tmid_and_bounds=True, + pick_comparison_by_eebls_snr=True, + exit_at_first_qc_pass_solution=True, + final_fit_baseline_duration_multiplier= + FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT, + use_adaptive_apertures=False, + adaptive_aperture_values=None, + adaptive_annulus_values=None, + fallback_sigma=np.nan, + run_fast_ultranest_before_final_run= + FAST_ULTRANEST_BEFORE_FINAL_RUN_DEFAULT, + run_final_fit_phase_residual_clip= + FINAL_FIT_PHASE_RESIDUAL_CLIP_DEFAULT, + run_final_residual_rejection=FINAL_RESIDUAL_REJECTION_DEFAULT, + save_dir=None, + planet_name=None, + observation_date=None, + use_ensemble_photometry_rather_than_single_comp=False, + maximum_number_of_ensemble_comparisons_for_transit= + TRANSIT_ENSEMBLE_MAX_COMPARISONS_DEFAULT, + exposure_times_seconds=None, + gain_e_per_adu=None, + use_exactly_the_comps_provided=False): + ranked_summaries = ranked_comparison_calibration_summaries( + comparison_calibration, + include_unvetted=use_exactly_the_comps_provided, + ) + if not ranked_summaries: + return { + 'ranked_summaries': [], + 'attempts': [], + 'selected_result': None, + } - if np.sum(~nanmask) <= 1: - log_info('No data left after filtering', warn=True) - return None, None, None + method = comparison_calibration['method'] + method_label = comparison_calibration.get('method_label', method) + aperture_index = comparison_calibration.get('a') + annulus_index = comparison_calibration.get('an') + if method == 'psf': + frame_count = target_psf_flux.shape[0] + else: + frame_count = aper_data['target'].shape[0] + exposure_times_array = None if exposure_times_seconds is None else np.asarray(exposure_times_seconds, dtype=float) + if exposure_times_array is not None and exposure_times_array.shape != times.shape: + exposure_times_array = None + if method == 'psf': + target_flux = np.asarray(target_psf_flux, dtype=float) + psf_flux_data = psf_flux_data_source(psf_data, psf_flux_data) + target_flux_error = ( + np.asarray(psf_noise_data.get('target'), dtype=float) + if isinstance(psf_noise_data, dict) and 'target' in psf_noise_data + else None + ) else: - arrayFinalFlux = arrayFinalFlux[~nanmask] - arrayNormUnc = arrayNormUnc[~nanmask] - arrayTimes = arrayTimes[~nanmask] - arrayJDTimes = arrayJDTimes[~nanmask] - arrayAirmass = arrayAirmass[~nanmask] + target_flux = np.asarray(aper_data['target'][:, aperture_index, annulus_index], dtype=float) + target_flux_error = ( + np.asarray(aper_data['target_unc'][:, aperture_index, annulus_index], dtype=float) + if isinstance(aper_data, dict) and 'target_unc' in aper_data + else None + ) + adaptive_summary = build_comparison_candidate_adaptive_summary( + comparison_calibration, + psf_data, + use_adaptive_apertures=use_adaptive_apertures, + adaptive_aperture_values=adaptive_aperture_values, + adaptive_annulus_values=adaptive_annulus_values, + fallback_sigma=fallback_sigma, + ) + field_image_keep_mask = np.asarray( + comparison_calibration.get('field_image_keep_mask', np.ones(times.shape[0], dtype=bool)), + dtype=bool, + ) + if field_image_keep_mask.shape != times.shape: + field_image_keep_mask = np.ones(times.shape[0], dtype=bool) + field_image_clip_diagnostic = None + if np.any(~field_image_keep_mask): + required_pairs = comparison_calibration.get('image_outlier_required_valid_pairs', 0) + sigma_threshold = comparison_calibration.get('image_outlier_sigma', COMPARISON_IMAGE_OUTLIER_SIGMA) + field_image_clip_diagnostic = build_time_rejection_diagnostic( + "Comparison-field image clip", + times, + field_image_keep_mask, + note=( + "Dropped frames flagged after comparison-star suitability clipping because every " + f"valid pairwise comparison was more than {sigma_threshold:.2f} sigma from its flat-line median " + f"(min valid pair count={required_pairs})." + ), + ) - # -----LM LIGHTCURVE FIT-------------------------------------- - prior = { - 'rprs': pDict['rprs'], # Rp/Rs - 'ars': pDict['aRs'], # a/Rs - 'per': pDict['pPer'], # Period [day] - 'inc': pDict['inc'], # Inclination [deg] - 'u0': ld[0], 'u1': ld[1], 'u2': ld[2], 'u3': ld[3], # limb darkening (nonlinear) - 'ecc': pDict['ecc'], # Eccentricity - 'omega': pDict['omega'], # Arg of periastron - 'tmid': pDict['midT'], # time of mid transit [day] - 'a1': arrayFinalFlux.mean(), # max() - arrayFinalFlux.min(), #mid Flux - 'a2': 0, # Flux lower bound - } + preflight_plans = [] + if use_ensemble_photometry_rather_than_single_comp: + if use_exactly_the_comps_provided: + active_keys = [summary.get('key') for summary in ranked_summaries if summary.get('key')] + ensemble_limit = len(active_keys) + else: + ensemble_limit = parse_maximum_number_of_ensemble_comparisons_for_transit( + maximum_number_of_ensemble_comparisons_for_transit + ) + active_keys = limited_ensemble_comparison_keys( + ranked_summaries, + ensemble_limit, + ) + if method == 'psf': + comp_flux_map = { + summary['key']: psf_flux_series_from_rows( + psf_flux_data[summary['key']], + np.asarray( + summary.get( + 'psf_quality_keep_mask', + psf_quality_mask_for_key( + psf_data, + summary['key'], + frame_count, + psf_flux_data=psf_flux_data, + ), + ), + dtype=bool, + ), + ) + for summary in ranked_summaries + if summary.get('key') in psf_flux_data + } + comp_error_map = { + summary['key']: mask_series_with_quality( + psf_noise_data[summary['key']], + np.asarray( + summary.get( + 'psf_quality_keep_mask', + psf_quality_mask_for_key( + psf_data, + summary['key'], + frame_count, + psf_flux_data=psf_flux_data, + ), + ), + dtype=bool, + ), + ) + for summary in ranked_summaries + if ( + isinstance(psf_noise_data, dict) + and summary.get('key') in psf_noise_data + ) + } + ensemble_flux, member_keys = build_absolute_comp_ensemble_flux( + comp_flux_map, + active_keys, + validity_mask_func=robust_flux_floor_mask, + ) + ensemble_flux_error = build_absolute_comp_ensemble_uncertainty( + comp_flux_map, + comp_error_map, + member_keys, + validity_mask_func=robust_flux_floor_mask, + ) + target_shape_mask = target_psf_shape_quality_mask(target_psf_quality_rows(psf_data, psf_flux_data=psf_flux_data)) + if target_shape_mask.shape[0] != frame_count: + target_shape_mask = np.ones(frame_count, dtype=bool) + candidate_target_flux = mask_series_with_quality(target_flux, target_shape_mask) + candidate_target_flux_error = ( + None + if target_flux_error is None + else mask_series_with_quality(target_flux_error, target_shape_mask) + ) + else: + comp_flux_map = { + summary['key']: mask_series_with_quality( + aper_data[summary['key']][:, aperture_index, annulus_index], + np.asarray( + summary.get( + 'psf_quality_keep_mask', + psf_quality_mask_for_key(psf_data, summary['key'], frame_count), + ), + dtype=bool, + ), + ) + for summary in ranked_summaries + if summary.get('key') in aper_data + } + comp_error_map = { + summary['key']: mask_series_with_quality( + aper_data[f"{summary['key']}_unc"][:, aperture_index, annulus_index], + np.asarray( + summary.get( + 'psf_quality_keep_mask', + psf_quality_mask_for_key(psf_data, summary['key'], frame_count), + ), + dtype=bool, + ), + ) + for summary in ranked_summaries + if summary.get('key') in aper_data and f"{summary['key']}_unc" in aper_data + } + ensemble_flux, member_keys = build_absolute_comp_ensemble_flux( + comp_flux_map, + active_keys, + validity_mask_func=valid_comparison_frame_mask, + ) + ensemble_flux_error = build_absolute_comp_ensemble_uncertainty( + comp_flux_map, + comp_error_map, + member_keys, + validity_mask_func=valid_comparison_frame_mask, + ) + target_shape_mask = np.ones(frame_count, dtype=bool) + candidate_target_flux = target_flux + candidate_target_flux_error = target_flux_error - arrayPhases = (arrayTimes - pDict['midT']) / prior['per'] - prior['tmid'] = pDict['midT'] + np.floor(arrayPhases).max() * prior['per'] - - upper = prior['tmid'] + np.abs(25 * pDict['midTUnc'] + np.floor(arrayPhases).max() * 25 * pDict['pPerUnc']) - lower = prior['tmid'] - np.abs(25 * pDict['midTUnc'] + np.floor(arrayPhases).max() * 25 * pDict['pPerUnc']) - - if upper > prior['tmid'] + 0.25 * prior['per']: - upper = prior['tmid'] + 0.25 * prior['per'] - if lower < prior['tmid'] - 0.25 * prior['per']: - lower = prior['tmid'] - 0.25 * prior['per'] - - if np.floor(arrayPhases).max() - np.floor(arrayPhases).min() == 0: - log_info("\nWarning:", warn=True) - log_info(" Estimated mid-transit time is not within the observations", warn=True) - log_info(" Check Period & Mid-transit time in inits.json. Make sure the uncertainties are not 0 or Nan.", warn=True) - log_info(f" obs start:{arrayTimes.min()}", warn=True) - log_info(f" obs end:{arrayTimes.max()}", warn=True) - log_info(f" tmid prior:{prior['tmid']}\n", warn=True) - - mybounds = { - 'rprs': [0, prior['rprs'] * 1.25], - 'tmid': [lower, upper], - 'inc': [prior['inc'] - 5, min(90, prior['inc'] + 5)], - 'a1': [0.5 * min(arrayFinalFlux), 2 * max(arrayFinalFlux)], - 'a2': [-1, 1] - } + exact_members_complete = ( + not use_exactly_the_comps_provided + or member_keys == active_keys + ) + if ensemble_flux is not None and member_keys and exact_members_complete: + candidate_frame_keep_mask = np.ones(times.shape[0], dtype=bool) + for summary in ranked_summaries: + if summary.get('key') not in set(member_keys): + continue + member_keep_mask = np.asarray( + summary.get('ensemble_frame_keep_mask', np.ones(times.shape[0], dtype=bool)), + dtype=bool, + ) + if member_keep_mask.shape == times.shape: + candidate_frame_keep_mask &= member_keep_mask + fit_mask = field_image_keep_mask & candidate_frame_keep_mask & target_shape_mask + if method == 'psf': + fit_mask &= robust_target_reference_flux_mask(candidate_target_flux, ensemble_flux) + else: + fit_mask &= ( + valid_comparison_frame_mask(candidate_target_flux) + & valid_comparison_frame_mask(ensemble_flux) + ) + fit_diagnostics = diagnose_lightcurve_fit_inputs( + times[fit_mask], + candidate_target_flux[fit_mask], + ensemble_flux[fit_mask], + airmass[fit_mask], + target_flux_error=None if candidate_target_flux_error is None else candidate_target_flux_error[fit_mask], + comp_flux_error=None if ensemble_flux_error is None else ensemble_flux_error[fit_mask], + enforce_relative_flux_max=False, + expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), + ) + preflight = build_comparison_candidate_preflight( + times[fit_mask], + jd_times[fit_mask], + airmass[fit_mask], + ld, + p_dict, + candidate_target_flux[fit_mask], + ensemble_flux[fit_mask], + target_flux_error=None if candidate_target_flux_error is None else candidate_target_flux_error[fit_mask], + comp_flux_error=None if ensemble_flux_error is None else ensemble_flux_error[fit_mask], + exposure_times_seconds=( + None if exposure_times_array is None else exposure_times_array[fit_mask] + ), + gain_e_per_adu=gain_e_per_adu, + adaptive_summary=adaptive_summary, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, + ) + ensemble_summary = { + 'comp_index': None, + 'key': 'ensemble', + 'label': f"Comparison ensemble ({len(member_keys)} comps)", + 'position': None, + 'aggregate_score': comparison_calibration.get('field_score', np.inf), + 'coverage_count': int(np.count_nonzero(valid_comparison_frame_mask(ensemble_flux))), + 'coverage_total_frame_count': int(frame_count), + 'coverage_reference_count': np.nan, + 'coverage_min_required_count': LIGHTCURVE_MIN_VALID_POINTS, + 'coverage_rejected': False, + 'ensemble_frame_rejected_count': int(np.count_nonzero(~candidate_frame_keep_mask)), + 'ensemble_frame_required_valid_pairs': len(member_keys), + 'ensemble_member_keys': member_keys, + } + preflight_plans.append({ + 'field_rank': 0, + 'summary': ensemble_summary, + 'ckey': None, + 'target_flux': candidate_target_flux, + 'comp_flux': ensemble_flux, + 'target_flux_error': candidate_target_flux_error, + 'comp_flux_error': ensemble_flux_error, + 'fit_mask': fit_mask, + 'candidate_frame_clip_diagnostic': None, + 'fit_diagnostics': fit_diagnostics, + 'preflight': preflight, + }) + log_info( + "Ensemble comparison photometry enabled: target fit will use " + f"{len(member_keys)} non-rejected comparison star(s) as a median normalized ensemble " + "rather than fitting each comparison star independently " + f"(configured maximum={ensemble_limit})." + ) + else: + if use_exactly_the_comps_provided: + log_info( + "Error: the fixed comparison ensemble could not be built with every supplied " + "comparison star; exact-comparison mode will not drop a member or fall back to " + "a different reference.", + error=True, + ) + else: + log_info( + "Warning: ensemble comparison photometry was enabled, but no usable non-rejected " + "comparison-star ensemble could be built; falling back to ranked single-comp fits.", + warn=True, + ) + + if ( + use_exactly_the_comps_provided + and use_ensemble_photometry_rather_than_single_comp + and not preflight_plans + ): + return { + 'ranked_summaries': ranked_summaries, + 'attempts': [], + 'selected_result': None, + 'selection_metric': 'exact_comparison_ensemble', + 'stopped_after_first_qc_pass': False, + 'stopped_after_promising_partial': False, + } - if np.isnan(arrayTimes).any() or np.isnan(arrayFinalFlux).any() or np.isnan(arrayNormUnc).any(): - log_info("\nWarning: NANs in time, flux or error", warn=True) + if not preflight_plans: + ranked_summaries_to_fit = ranked_summaries + else: + ranked_summaries_to_fit = [] + + for field_rank, comp_summary in enumerate(ranked_summaries_to_fit): + comp_index = comp_summary['comp_index'] + ckey = comp_summary.get('key', f"comp{comp_index + 1}") + comp_quality_mask = np.asarray( + comp_summary.get( + 'psf_quality_keep_mask', + psf_quality_mask_for_key( + psf_data, + ckey, + frame_count, + psf_flux_data=psf_flux_data if method == 'psf' else None, + ), + ), + dtype=bool, + ) + if comp_quality_mask.shape[0] != frame_count: + comp_quality_mask = psf_quality_mask_for_key( + psf_data, + ckey, + frame_count, + psf_flux_data=psf_flux_data if method == 'psf' else None, + ) + if method == 'psf': + target_shape_mask = target_psf_shape_quality_mask( + target_psf_quality_rows(psf_data, psf_flux_data=psf_flux_data), + psf_quality_rows_for_key(psf_data, ckey, psf_flux_data=psf_flux_data), + ) + if target_shape_mask.shape[0] != frame_count: + target_shape_mask = np.ones(frame_count, dtype=bool) + candidate_target_flux = mask_series_with_quality(target_flux, target_shape_mask) + candidate_target_flux_error = ( + None + if target_flux_error is None + else mask_series_with_quality(target_flux_error, target_shape_mask) + ) + comp_flux = psf_flux_series_from_rows(psf_flux_data[ckey], comp_quality_mask) + comp_flux_error = ( + mask_series_with_quality(psf_noise_data[ckey], comp_quality_mask) + if isinstance(psf_noise_data, dict) and ckey in psf_noise_data + else None + ) + else: + target_shape_mask = np.ones(frame_count, dtype=bool) + candidate_target_flux = target_flux + candidate_target_flux_error = target_flux_error + comp_flux = mask_series_with_quality(aper_data[ckey][:, aperture_index, annulus_index], comp_quality_mask) + comp_flux_error = ( + mask_series_with_quality( + aper_data[f"{ckey}_unc"][:, aperture_index, annulus_index], + comp_quality_mask, + ) + if f"{ckey}_unc" in aper_data + else None + ) - myfit = lc_fitter( - arrayTimes, - arrayFinalFlux, - arrayNormUnc, - arrayAirmass, - prior, - mybounds, - jd_times=arrayJDTimes, - mode='lm' + candidate_frame_keep_mask = np.asarray( + comp_summary.get('ensemble_frame_keep_mask', np.ones(times.shape[0], dtype=bool)), + dtype=bool, + ) + if candidate_frame_keep_mask.shape != times.shape: + candidate_frame_keep_mask = np.ones(times.shape[0], dtype=bool) + candidate_frame_clip_diagnostic = None + candidate_frame_diagnostic_keep_mask = candidate_frame_keep_mask | ~field_image_keep_mask + if np.any(~candidate_frame_diagnostic_keep_mask): + required_pairs = comp_summary.get( + 'ensemble_frame_required_valid_pairs', + COMPARISON_CANDIDATE_FRAME_OUTLIER_MIN_VALID_PAIRS, + ) + sigma_threshold = comp_summary.get('ensemble_frame_sigma', COMPARISON_IMAGE_OUTLIER_SIGMA) + candidate_frame_clip_diagnostic = build_time_rejection_diagnostic( + "Comparison-candidate intercomparison clip", + times, + candidate_frame_diagnostic_keep_mask, + note=( + "Dropped frames where this comparison star disagreed with the peer-star " + "intercomparison reference " + f"before target fitting; same-direction pairwise majority exceeded {sigma_threshold:.2f} sigma " + f"(min confirming pair count={required_pairs})." + ), + ) + + fit_mask = field_image_keep_mask & candidate_frame_keep_mask & comp_quality_mask & target_shape_mask + if method == 'psf': + fit_mask &= robust_target_reference_flux_mask(candidate_target_flux, comp_flux) + else: + fit_mask &= valid_comparison_frame_mask(candidate_target_flux) & valid_comparison_frame_mask(comp_flux) + + fit_diagnostics = diagnose_lightcurve_fit_inputs( + times[fit_mask], + candidate_target_flux[fit_mask], + comp_flux[fit_mask], + airmass[fit_mask], + target_flux_error=None if candidate_target_flux_error is None else candidate_target_flux_error[fit_mask], + comp_flux_error=None if comp_flux_error is None else comp_flux_error[fit_mask], + enforce_relative_flux_max=False, + expected_transit_depth=expected_transit_depth_from_planet_dict(p_dict), + ) + preflight = build_comparison_candidate_preflight( + times[fit_mask], + jd_times[fit_mask], + airmass[fit_mask], + ld, + p_dict, + candidate_target_flux[fit_mask], + comp_flux[fit_mask], + target_flux_error=None if candidate_target_flux_error is None else candidate_target_flux_error[fit_mask], + comp_flux_error=None if comp_flux_error is None else comp_flux_error[fit_mask], + exposure_times_seconds=( + None if exposure_times_array is None else exposure_times_array[fit_mask] + ), + gain_e_per_adu=gain_e_per_adu, + adaptive_summary=adaptive_summary, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, + ) + preflight_plans.append({ + 'field_rank': field_rank, + 'summary': comp_summary, + 'ckey': ckey, + 'target_flux': candidate_target_flux, + 'comp_flux': comp_flux, + 'target_flux_error': candidate_target_flux_error, + 'comp_flux_error': comp_flux_error, + 'fit_mask': fit_mask, + 'candidate_frame_clip_diagnostic': candidate_frame_clip_diagnostic, + 'fit_diagnostics': fit_diagnostics, + 'preflight': preflight, + }) + + ranked_preflight_plans = rank_comparison_candidate_preflight_plans(preflight_plans) + log_comparison_candidate_preflight_order(preflight_plans, ranked_preflight_plans) + + attempts = [] + stopped_after_first_qc_pass = False + stopped_after_promising_partial = False + for rank, plan in enumerate(ranked_preflight_plans): + comp_summary = plan['summary'] + comp_index = comp_summary['comp_index'] + ckey = plan['ckey'] + candidate_target_flux = plan.get('target_flux', target_flux) + comp_flux = plan['comp_flux'] + candidate_target_flux_error = plan.get('target_flux_error') + comp_flux_error = plan.get('comp_flux_error') + fit_mask = plan['fit_mask'] + candidate_frame_clip_diagnostic = plan.get('candidate_frame_clip_diagnostic') + fit_diagnostics = plan['fit_diagnostics'] + preflight = plan.get('preflight') or {} + log_comparison_candidate_evaluation_start( + comp_summary, + rank, + len(ranked_preflight_plans), + method_label, + fit_diagnostics, + ) + log_info( + " Full reduction starting. Optional out-of-transit baseline detrending is " + f"{'enabled' if detrend_on_outoftransit_baseline else 'disabled'}." + ) + final_reduction = finalize_comparison_candidate_full_reduction( + times[fit_mask], + candidate_target_flux[fit_mask], + comp_flux[fit_mask], + airmass[fit_mask], + ld, + p_dict, + jd_times=jd_times[fit_mask], + target_flux_error=None if candidate_target_flux_error is None else candidate_target_flux_error[fit_mask], + comp_flux_error=None if comp_flux_error is None else comp_flux_error[fit_mask], + exposure_times_seconds=( + None if exposure_times_array is None else exposure_times_array[fit_mask] + ), + gain_e_per_adu=gain_e_per_adu, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + detrend_on_outoftransit_baseline=detrend_on_outoftransit_baseline, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_to_initialize_tmid_and_bounds, + plot_time_range=plot_time_range, + baseline_duration_multiplier=final_fit_baseline_duration_multiplier, + adaptive_summary=adaptive_summary, + run_fast_ultranest_before_final_run=run_fast_ultranest_before_final_run, + run_final_fit_phase_residual_clip=run_final_fit_phase_residual_clip, + run_final_residual_rejection=run_final_residual_rejection, + precomputed_candidate_series=preflight.get('prepared_series'), + ) + fit_result = final_reduction.get('fit') if final_reduction.get('applied') else None + tflux_fit = final_reduction.get('good_target_flux') + cflux_fit = final_reduction.get('good_comp_flux') + tflux_fit_error = final_reduction.get('good_target_flux_error') + cflux_fit_error = final_reduction.get('good_comp_flux_error') + fit_diagnostics = ensure_lightcurve_fit_failure_reason( + fit_diagnostics, + fit_result, + failed_stage='full_candidate_reduction', + failure_reason=final_reduction.get( + 'failure_reason', + "the raw comparison-candidate photometry did not converge to a usable fully reduced solution.", + ), + ) + if fit_result is None: + log_info( + f" {comp_summary.get('label', 'comparison candidate')}: " + "the raw comparison-candidate light curve did not converge to a usable fully reduced fit." + ) + selection_fit = fit_result + fast_binning = final_reduction.get('fast_ultranest_binning') or {} + fast_binned_ultranest = bool(fast_binning.get('applied')) + residual_scatter = extract_lightcurve_fit_residual_scatter(selection_fit) + target_comp_scatter_value = target_comp_flux_scatter(tflux_fit, cflux_fit) + projected_full_scatter = fitted_lightcurve_scatter_on_dataset( + selection_fit, + final_reduction.get('good_times'), + final_reduction.get('good_flux'), + final_reduction.get('good_airmass'), + ) + if fast_binned_ultranest and np.isfinite(projected_full_scatter): + selection_scatter = projected_full_scatter + selection_scatter_basis = ( + "fast-binned UltraNest model residual scatter evaluated on full unbinned light curve" + ) + else: + selection_scatter = residual_scatter + selection_scatter_basis = ( + "full-resolution UltraNest model residual scatter" + if not fast_binned_ultranest + else "fast-binned UltraNest model residual scatter" + ) + target_model_scatter_basis = ( + "fast-binned UltraNest model residual scatter" + if fast_binned_ultranest + else "full-resolution UltraNest model residual scatter" + ) + transit_qc_failure_reason = lightcurve_fit_transit_qc_failure_reason(selection_fit) + if transit_qc_failure_reason is not None: + fit_diagnostics = dict(fit_diagnostics) + fit_diagnostics.update({ + 'failed_stage': 'transit_qc', + 'failure_reason': transit_qc_failure_reason, + }) + external_filter_diagnostics = [] + if field_image_clip_diagnostic is not None: + external_filter_diagnostics.append(field_image_clip_diagnostic) + if candidate_frame_clip_diagnostic is not None: + external_filter_diagnostics.append(candidate_frame_clip_diagnostic) + if external_filter_diagnostics: + attached_fit_ids = set() + for fit_candidate in (fit_result, final_reduction.get('fit'), selection_fit): + if fit_candidate is None or id(fit_candidate) in attached_fit_ids: + continue + attached_fit_ids.add(id(fit_candidate)) + for diagnostic in reversed(external_filter_diagnostics): + prepend_lightcurve_filter_diagnostic(fit_candidate, diagnostic) + + attempt = { + 'rank': rank, + 'field_rank': plan.get('field_rank'), + 'comp_index': comp_index, + 'ckey': ckey, + 'label': comp_summary.get('label', 'comparison candidate'), + 'position': comp_summary.get('position'), + 'aggregate_score': comp_summary.get('aggregate_score', np.inf), + 'coverage_count': comp_summary.get('coverage_count', 0), + 'coverage_total_frame_count': comp_summary.get('coverage_total_frame_count', 0), + 'coverage_reference_count': comp_summary.get('coverage_reference_count', np.nan), + 'coverage_min_required_count': comp_summary.get('coverage_min_required_count', 0), + 'coverage_rejected': comp_summary.get('coverage_rejected', False), + 'ensemble_frame_rejected_count': comp_summary.get('ensemble_frame_rejected_count', 0), + 'ensemble_frame_required_valid_pairs': comp_summary.get('ensemble_frame_required_valid_pairs', 0), + 'ensemble_member_keys': list(comp_summary.get('ensemble_member_keys') or []), + 'fit': selection_fit, + 'provisional_fit': None, + 'full_reduction_fit': final_reduction.get('fit'), + 'good_times': final_reduction.get('good_times'), + 'good_flux': final_reduction.get('good_flux'), + 'good_unc': final_reduction.get('good_unc'), + 'good_airmass': final_reduction.get('good_airmass'), + 'good_jd_times': final_reduction.get('good_jd_times'), + 'good_exposure_times_seconds': final_reduction.get('good_exposure_times_seconds'), + 'good_target_flux_error': tflux_fit_error, + 'good_comp_flux_error': cflux_fit_error, + 'tflux_fit': tflux_fit, + 'cflux_fit': cflux_fit, + 'tflux_fit_error': tflux_fit_error, + 'cflux_fit_error': cflux_fit_error, + 'source_indices': final_reduction.get('source_indices'), + 'duration_samples': final_reduction.get('duration_samples'), + 'data_highres': final_reduction.get('data_highres'), + 'fit_diagnostics': fit_diagnostics, + 'eebls_snr': extract_lightcurve_fit_eebls_snr(selection_fit), + 'transit_delta_bic': extract_lightcurve_fit_transit_delta_bic(selection_fit), + 'residual_scatter': residual_scatter, + 'target_model_scatter_basis': target_model_scatter_basis, + 'projected_full_residual_scatter': projected_full_scatter, + 'selection_scatter': selection_scatter, + 'selection_scatter_basis': selection_scatter_basis, + 'target_comp_scatter': target_comp_scatter_value, + 'ktmf_metric': extract_lightcurve_fit_ktmf_metric(selection_fit), + 'ktmf_contributions': extract_lightcurve_fit_ktmf_contributions(selection_fit), + 'fit_point_count': 0 if tflux_fit is None else int(len(np.asarray(tflux_fit, dtype=float))), + 'failure_reason': fit_diagnostics.get('failure_reason'), + 'parameter_summary': summarize_lightcurve_fit_parameters(selection_fit), + 'transit_qc_status': getattr(selection_fit, 'transit_qc_status', None), + 'transit_qc_summary': getattr(selection_fit, 'transit_qc_summary', None), + 'rejected_by_transit_qc': final_reduction.get('applied', False) and transit_qc_failure_reason is not None, + 'selected': False, + 'selection_reason': None, + 'search_stopped_after_qc_pass': False, + 'search_stopped_after_promising_partial': False, + 'failed_run_dir': None, + 'final_output_dir': None, + 'full_reduction_applied': final_reduction.get('applied', False), + 'full_reduction_note': final_reduction.get('note'), + 'fast_ultranest_binning': fast_binning, + 'skip_airmass_fit': final_reduction.get('skip_airmass_fit', False), + 'airmass_skip_note': final_reduction.get('airmass_skip_note'), + 'preflight_coverage_priority': preflight.get('coverage_priority'), + 'preflight_scout_score': (preflight.get('scout') or {}).get('score', np.nan), + } + if final_reduction.get('applied') and selection_fit is not None and save_dir is not None: + final_output_dir = save_comparison_candidate_full_reduction_outputs( + save_dir, + None, + selection_fit, + p_dict, + observation_date, + comp_index, + comp_coords=comp_summary.get('position'), + min_aperture=(0 if comparison_calibration['method'] == 'psf' else comparison_calibration.get('aper')), + min_annulus=comparison_calibration.get('annulus'), + adaptive_summary=adaptive_summary, + method_label=comparison_calibration.get('method_label'), + selection_summary={ + 'ktmf_metric': extract_lightcurve_fit_ktmf_metric(selection_fit), + 'transit_delta_bic': extract_lightcurve_fit_transit_delta_bic(selection_fit), + 'eebls_snr': extract_lightcurve_fit_eebls_snr(selection_fit), + 'transit_qc_status': getattr(selection_fit, 'transit_qc_status', None), + 'transit_qc_summary': getattr(selection_fit, 'transit_qc_summary', None), + }, + duration_samples=final_reduction.get('duration_samples'), + data_highres=final_reduction.get('data_highres'), + ) + if final_output_dir is not None: + attempt['final_output_dir'] = str(final_output_dir) + if transit_qc_failure_reason is not None: + attempt['selection_reason'] = ( + "rejected: transit detection QC flagged this comparison as a poor transit candidate" + ) + archive_dir = archive_failed_comparison_fit( + save_dir, + planet_name, + observation_date, + attempt, + method_label=comparison_calibration.get('method_label'), + ) + if archive_dir is not None: + attempt['failed_run_dir'] = str(archive_dir) + log_comparison_candidate_evaluation_result(attempt) + attempts.append(attempt) + if ( + exit_at_first_qc_pass_solution + and attempt.get('fit') is not None + and attempt.get('full_reduction_applied', False) + and lightcurve_fit_transit_qc_passed(selection_fit) + ): + attempt['search_stopped_after_qc_pass'] = True + stopped_after_first_qc_pass = True + log_info( + "Stopping comparison-star candidate search after the first transit-QC PASS fit " + f"({attempt['label']})." + ) + break + if ( + exit_at_first_qc_pass_solution + and should_stop_after_promising_partial_comparison_attempt(attempt) + ): + attempt['search_stopped_after_promising_partial'] = True + stopped_after_promising_partial = True + log_info( + "Stopping comparison-star candidate search after a promising partial-coverage " + f"MARGINAL fit ({attempt['label']}); proceeding to selected full-resolution confirmation." + ) + break + + selected_result = None + completed_attempts = [ + attempt + for attempt in attempts + if ( + attempt.get('fit') is not None + and attempt.get('full_reduction_applied', False) + ) + ] + successful_attempts = [ + attempt + for attempt in completed_attempts + if not attempt.get('rejected_by_transit_qc', False) + ] + first_qc_pass_attempt = next( + ( + attempt for attempt in attempts + if attempt.get('search_stopped_after_qc_pass', False) + ), + None, + ) + first_promising_partial_attempt = next( + ( + attempt for attempt in attempts + if attempt.get('search_stopped_after_promising_partial', False) + ), + None, ) + selection_metric = 'ktmf' + fallback_to_qc_rejected = False + if first_qc_pass_attempt is not None: + selected_result = first_qc_pass_attempt + selection_metric = 'first_qc_pass' + elif first_promising_partial_attempt is not None: + selected_result = first_promising_partial_attempt + selection_metric = 'promising_partial' + elif successful_attempts: + selected_result, selection_metric = select_preferred_comparison_attempt( + successful_attempts, + pick_comparison_by_eebls_snr=pick_comparison_by_eebls_snr, + ) + else: + qc_rejected_attempts = [ + attempt for attempt in completed_attempts + if attempt.get('rejected_by_transit_qc', False) + ] + selected_result, selection_metric = select_preferred_comparison_attempt( + qc_rejected_attempts, + pick_comparison_by_eebls_snr=pick_comparison_by_eebls_snr, + ) + fallback_to_qc_rejected = selected_result is not None + + if selected_result is not None: + selected_result['selected'] = True + selected_result['selected_despite_transit_qc'] = fallback_to_qc_rejected + for attempt in attempts: + record_comparison_attempt_selection_pass_metrics(attempt) + selected_ktmf_metric = selected_result.get('selection_pass_ktmf_metric', np.nan) + selected_transit_delta_bic = selected_result.get('selection_pass_transit_delta_bic', np.nan) + selected_eebls_snr = selected_result.get('selection_pass_eebls_snr', np.nan) + selected_scatter_adjusted_ktmf = selected_result.get('scatter_adjusted_ktmf_metric', np.nan) + selected_combined_quality_ktmf = selected_result.get('combined_quality_ktmf_metric', np.nan) + + for attempt in attempts: + if attempt is selected_result: + if fallback_to_qc_rejected: + attempt['selection_reason'] = ( + "selected as best available fallback: all completed comparison-star " + "target fits were rejected by transit QC" + ) + if ( + selection_metric in ('ktmf', 'ktmf_scatter', 'ktmf_combined_quality') + and np.isfinite(selected_ktmf_metric) + ): + attempt['selection_reason'] += ( + f"; this candidate had the highest KTMF/projected-scatter score " + f"among candidates with projected scatter " + f"<= {COMPARISON_SELECTION_MAX_SCATTER_MULTIPLIER:.1f}x the lowest scatter" + ) + if np.isfinite(selected_combined_quality_ktmf): + attempt['selection_reason'] += ( + f"; KTMF/projected-scatter score={selected_combined_quality_ktmf:.2f}; " + f"raw selection-pass KTMF={format_ktmf_metric(selected_ktmf_metric)}" + ) + elif selection_metric == 'eebls_snr' and np.isfinite(selected_eebls_snr): + attempt['selection_reason'] += ( + f"; this candidate had the highest selection-pass EEBLS SNR " + f"({selected_eebls_snr:.2f})" + ) + elif np.isfinite(selected_transit_delta_bic): + attempt['selection_reason'] += ( + "; this candidate had the strongest selection-pass transit-vs-flat Delta BIC " + f"({format_transit_delta_bic(selected_transit_delta_bic)})" + ) + elif attempt.get('search_stopped_after_qc_pass', False): + attempt['selection_reason'] = ( + "selected: first completed comparison-star candidate with PASS transit QC" + ) + elif attempt.get('search_stopped_after_promising_partial', False): + attempt['selection_reason'] = ( + "selected: first partial-coverage comparison-star candidate with promising " + "MARGINAL transit diagnostics" + ) + elif ( + selection_metric in ('ktmf', 'ktmf_scatter', 'ktmf_combined_quality') + and np.isfinite(selected_ktmf_metric) + ): + attempt['selection_reason'] = ( + "selected: highest KTMF/projected-scatter score among comparison-star calibration " + f"candidates with projected scatter <= " + f"{COMPARISON_SELECTION_MAX_SCATTER_MULTIPLIER:.1f}x the lowest scatter" + ) + if np.isfinite(selected_combined_quality_ktmf): + attempt['selection_reason'] += ( + f"; KTMF/projected-scatter score={selected_combined_quality_ktmf:.2f}; " + f"raw selection-pass KTMF={format_ktmf_metric(selected_ktmf_metric)}" + ) + elif selection_metric == 'eebls_snr' and np.isfinite(selected_eebls_snr): + attempt['selection_reason'] = ( + "selected: highest selection-pass EEBLS SNR among the evaluated " + "comparison-star calibration candidates" + ) + else: + attempt['selection_reason'] = ( + "selected: strongest selection-pass transit-vs-flat Delta BIC among " + "the evaluated comparison-star calibration candidates" + ) + continue + if ( + attempt.get('fit') is not None + and attempt.get('full_reduction_applied', False) + and (not attempt.get('rejected_by_transit_qc', False) or fallback_to_qc_rejected) + ): + if attempt.get('scatter_gate_passed') is False: + selection_scatter_basis = attempt.get('selection_scatter_basis') or "selection scatter" + attempt['selection_reason'] = ( + f"not selected: {selection_scatter_basis} " + f"{format_residual_scatter(attempt.get('selection_scatter', np.nan))} " + f"exceeded {COMPARISON_SELECTION_MAX_SCATTER_MULTIPLIER:.1f}x the lowest candidate scatter " + f"({format_residual_scatter(attempt.get('scatter_gate_lowest_residual_scatter', np.nan))}); " + "excluded before KTMF ranking" + ) + elif ( + selection_metric in ('ktmf', 'ktmf_scatter', 'ktmf_combined_quality') + and np.isfinite(selected_ktmf_metric) + ): + attempt_score = attempt.get('combined_quality_ktmf_metric', np.nan) + selected_score = selected_combined_quality_ktmf + score_text = ( + f"KTMF/projected-scatter score {attempt_score:.2f} was lower than the selected " + f"{selected_score:.2f}" + if np.isfinite(attempt_score) and np.isfinite(selected_score) + else "selection-pass KTMF was lower than the selected candidate" + ) + attempt['selection_reason'] = ( + "not selected: " + f"{score_text}; raw selection-pass KTMF=" + f"{format_ktmf_metric(attempt.get('selection_pass_ktmf_metric', np.nan))}" + ) + elif selection_metric == 'first_qc_pass': + attempt['selection_reason'] = ( + "not selected: search stopped after the first comparison-star candidate " + "with PASS transit QC" + ) + elif selection_metric == 'promising_partial': + attempt['selection_reason'] = ( + "not selected: search stopped after the first partial-coverage comparison-star " + "candidate with promising MARGINAL transit diagnostics" + ) + elif selection_metric == 'eebls_snr' and np.isfinite(selected_eebls_snr): + if np.isfinite(attempt.get('selection_pass_eebls_snr', np.nan)): + attempt['selection_reason'] = ( + "not selected: selection-pass EEBLS SNR " + f"{attempt['selection_pass_eebls_snr']:.2f} was lower than the selected " + f"{selected_eebls_snr:.2f}" + ) + else: + attempt['selection_reason'] = ( + "not selected: no finite EEBLS SNR was available for this candidate" + ) + else: + attempt['selection_reason'] = ( + "not selected: selection-pass transit-vs-flat Delta BIC " + f"{format_transit_delta_bic(attempt.get('selection_pass_transit_delta_bic', np.nan))} " + "was lower than the selected " + f"{format_transit_delta_bic(selected_transit_delta_bic)}" + ) + + selected_fit = selected_result.get('fit') + if selected_fit is not None: + full_resolution_refit_applied = False + full_resolution_refit = refit_selected_fast_comparison_on_full_lightcurve( + selected_result, + p_dict, + skip_airmass_fit=bool(selected_result.get('skip_airmass_fit', False)), + airmass_skip_note=selected_result.get('airmass_skip_note'), + detrend_on_outoftransit_baseline=detrend_on_outoftransit_baseline, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + plot_time_range=plot_time_range, + duration_prior=build_single_transit_duration_prior(p_dict), + run_final_residual_rejection=run_final_residual_rejection, + ) + if full_resolution_refit is not None: + full_resolution_refit_applied = True + selected_result['fit'], selected_result['good_flux'], selected_result['good_unc'] = full_resolution_refit + selected_result['full_reduction_note'] = ( + "selected candidate rerun on the full-resolution light curve after fast UltraNest search." + ) + else: + selected_result['fit'] = extend_selected_comparison_live_points_if_needed(selected_fit) + selected_result['full_reduction_fit'] = selected_result['fit'] + if ( + full_resolution_refit_applied + or getattr(selected_result['fit'], 'sparse_posterior_live_point_extension_applied', False) + ): + annotate_transit_detection_qc(selected_result['fit']) + selected_result['eebls_snr'] = extract_lightcurve_fit_eebls_snr(selected_result['fit']) + selected_result['transit_delta_bic'] = extract_lightcurve_fit_transit_delta_bic(selected_result['fit']) + selected_result['residual_scatter'] = extract_lightcurve_fit_residual_scatter(selected_result['fit']) + final_target_comp_scatter = target_comp_flux_scatter( + selected_result.get('good_target_flux'), + selected_result.get('good_comp_flux'), + ) + if np.isfinite(final_target_comp_scatter): + selected_result['target_comp_scatter'] = final_target_comp_scatter + selected_result['ktmf_metric'] = extract_lightcurve_fit_ktmf_metric(selected_result['fit']) + selected_result['ktmf_contributions'] = extract_lightcurve_fit_ktmf_contributions(selected_result['fit']) + selected_result['parameter_summary'] = summarize_lightcurve_fit_parameters(selected_result['fit']) + selected_result['transit_qc_status'] = getattr(selected_result['fit'], 'transit_qc_status', None) + selected_result['transit_qc_summary'] = getattr(selected_result['fit'], 'transit_qc_summary', None) + update_selected_comparison_final_refit_note(selected_result) + data_highres, duration_samples = estimate_transit_duration_samples_from_fit(selected_result['fit']) + selected_result['data_highres'] = data_highres + selected_result['duration_samples'] = duration_samples + if save_dir is not None: + final_output_dir = save_comparison_candidate_full_reduction_outputs( + save_dir, + None, + selected_result['fit'], + p_dict, + observation_date, + selected_result['comp_index'], + comp_coords=selected_result.get('position'), + min_aperture=(0 if comparison_calibration['method'] == 'psf' else comparison_calibration.get('aper')), + min_annulus=comparison_calibration.get('annulus'), + adaptive_summary=adaptive_summary, + method_label=comparison_calibration.get('method_label'), + selection_summary={ + 'ktmf_metric': selected_result.get('ktmf_metric', np.nan), + 'transit_delta_bic': selected_result.get('transit_delta_bic', np.nan), + 'eebls_snr': selected_result.get('eebls_snr', np.nan), + 'transit_qc_status': selected_result.get('transit_qc_status'), + 'transit_qc_summary': selected_result.get('transit_qc_summary'), + }, + duration_samples=selected_result.get('duration_samples'), + data_highres=selected_result.get('data_highres'), + ) + if final_output_dir is not None: + selected_result['final_output_dir'] = str(final_output_dir) + + if use_exactly_the_comps_provided: + selection_metric = ( + 'exact_comparison_ensemble' + if use_ensemble_photometry_rather_than_single_comp + else 'exact_single_comparison' + ) + if selected_result is not None: + selected_result['selection_reason'] = ( + 'selected: used every supplied comparison as one fixed ensemble without vetting' + if use_ensemble_photometry_rather_than_single_comp + else 'selected: used the one supplied comparison without vetting or alternatives' + ) - return myfit, f1, f2 + for attempt in attempts: + if selected_result is not None and attempt is selected_result: + continue + clear_fit_ultranest_resume_state(attempt.get('fit')) + full_reduction_fit = attempt.get('full_reduction_fit') + if full_reduction_fit is not attempt.get('fit'): + clear_fit_ultranest_resume_state(full_reduction_fit) + + return { + 'ranked_summaries': ranked_summaries, + 'attempts': attempts, + 'selected_result': selected_result, + 'selection_metric': selection_metric, + 'stopped_after_first_qc_pass': stopped_after_first_qc_pass, + 'stopped_after_promising_partial': stopped_after_promising_partial, + } def parse_args(): @@ -1814,12 +32001,42 @@ def parse_args(): "Can be used as an additional argument with -rt (--realtime), -red (--reduce), " "-pre (--prereduced), and -phot (--photometry)." "Do not combine with the -ov, --override argument.") + parser.add_argument('--use-nextastro-astrometry', + action='store_true', + help="Use NextAstro's astrometry service (https://astrometry.nextastro.org/) instead of nova.astrometry.net for plate solving.") + parser.add_argument('--use-nextastro-variability-server', + action='store_true', + help="Use NextAstro's variability server for a batch comparison-star variability check. " + "If the service returns an error, EXOTIC falls back to individual VSX checks.") + parser.add_argument('--non-interactive-run', + action='store_true', + help="Avoid interactive prompts for invalid target RA/Dec values, target pixel-coordinate " + "mismatches, and unrecognized limb-darkening filters. Invalid initialization-file " + "RA/Dec values use NASA Exoplanet Archive coordinates when available, otherwise " + "the run aborts. Pixel mismatches use an automatic fallback; unrecognized filters " + "abort unless wl_min and wl_max are provided.") + parser.add_argument('--multiprocess-transformations', + type=int, + default=None, + help="Use multiprocessing for frame alignment and fallback image transformations. " + "Provide an integer number of processes to use.") + parser.add_argument('--multiprocess-lightcurve-fits', + type=int, + default=None, + help="Use multiprocessing while evaluating candidate lightcurve fits. " + "Provide an integer number of processes to use.") return parser.parse_args() -def main(): +def _main_impl(): # command line args args = parse_args() + if args.multiprocess_transformations is not None and args.multiprocess_transformations < 1: + raise ValueError("--multiprocess-transformations requires an integer greater than 0.") + if args.multiprocess_lightcurve_fits is not None and args.multiprocess_lightcurve_fits < 1: + raise ValueError("--multiprocess-lightcurve-fits requires an integer greater than 0.") + configure_windows_multiprocessing_main_spec() + validate_ultranest_mpi_runtime() log.debug("*************************") log.debug("EXOTIC reduction log file") @@ -1878,6 +32095,7 @@ def main(): init_path, userpDict = inputs_obj.search_init(args.realtime, userpDict) exotic_infoDict, userpDict['pName'] = inputs_obj.real_time(userpDict['pName']) + configure_runtime_logging(output_dir=exotic_infoDict.get('save')) while True: carry_on = user_input(f"\nType continue after the first image has been taken and saved: ", type_=str) @@ -1893,11 +32111,17 @@ def main(): ax.set_ylabel('Normalized Flux') ax.set_xlabel('Time (JD)') - anim = FuncAnimation(fig, realTimeReduce, fargs=(userpDict['pName'], userpDict, exotic_infoDict, ax), interval=15000) + anim = FuncAnimation( + fig, + realTimeReduce, + fargs=(userpDict['pName'], userpDict, exotic_infoDict, ax, args.use_nextastro_astrometry, args.multiprocess_transformations), + interval=15000 + ) plt.show() # ----USER INPUTS---------------------------------------------------------- else: + reduction_stage_timer = ReductionStageTimer() log_info("\n**************************") log_info("Complete Reduction Routine") log_info("**************************") @@ -1906,6 +32130,18 @@ def main(): generalDark, generalBias, generalFlat = np.empty(shape=(0, 0)), np.empty(shape=(0, 0)), np.empty(shape=(0, 0)) demosaic_fmt = None demosaic_out = None + precheck_inputfile_count = None + post_wcs_inputfile_count = None + post_target_wcs_inputfile_count = None + post_pointing_inputfile_count = None + dropped_wcs_files = [] + dropped_target_wcs_files = [] + dropped_pointing_files = [] + ignore_header_wcs = False + bad_wcs_threshold_fraction = np.nan + pointing_rejection_sigma = np.nan + detect_bad_pixels_before_photometry = None + bad_pixel_reference = None if isinstance(args.reduce, str): fitsortext = 1 @@ -1936,16 +32172,421 @@ def main(): exotic_infoDict, userpDict['pName'] = inputs_obj.complete_red(userpDict['pName']) else: exotic_infoDict, userpDict['pName'] = inputs_obj.prereduced(userpDict['pName']) - - # Make a temp directory of helpful files - Path(Path(exotic_infoDict['save']) / "temp").mkdir(exist_ok=True) - + for motion_key in ('dist', 'pm_ra', 'pm_dec'): + current_motion_value = userpDict.get(motion_key) + if current_motion_value is None or (isinstance(current_motion_value, str) and not current_motion_value.strip()): + header_motion_value = exotic_infoDict.get(motion_key) + if header_motion_value is not None: + userpDict[motion_key] = header_motion_value + configure_runtime_logging(output_dir=exotic_infoDict.get('save')) + disable_vertical_flux_normalization = is_vertical_flux_normalization_disabled( + exotic_infoDict.get('disable_vertical_flux_normalization', False) + ) + detrend_on_outoftransit_baseline = is_out_of_transit_baseline_detrending_enabled( + exotic_infoDict.get('detrend_on_outoftransit_baseline', True) + ) + final_fit_baseline_duration_multiplier = get_final_fit_baseline_duration_multiplier( + exotic_infoDict.get( + 'final_fit_baseline_duration_multiplier', + FINAL_FIT_BASELINE_DURATION_MULTIPLIER_DEFAULT, + ) + ) + use_eebls_tmid_initializer = should_use_eebls_to_initialize_tmid_and_bounds( + exotic_infoDict.get('use_eebls_to_initialize_tmid_and_bounds', 'y') + ) + pick_comparison_by_eebls_snr = should_pick_comparison_by_eebls_snr( + exotic_infoDict.get('pick_comparison_by_eebls_snr', 'y') + ) + use_impactparameter_rather_than_inclination_to_fit = ( + should_use_impactparameter_rather_than_inclination_to_fit( + exotic_infoDict.get('use_impactparameter_rather_than_inclination_to_fit', 'y') + ) + ) + run_fast_ultranest_before_final_run = should_run_fast_ultranest_before_final_run( + exotic_infoDict.get( + 'run_fast_ultranest_before_final_run', + FAST_ULTRANEST_BEFORE_FINAL_RUN_DEFAULT, + ) + ) + run_final_residual_rejection = should_run_final_residual_rejection( + exotic_infoDict.get( + 'run_final_residual_rejection', + exotic_infoDict.get( + 'Run Final Residual Rejection and Extra UltraNest Run', + FINAL_RESIDUAL_REJECTION_DEFAULT, + ), + ) + ) + run_final_fit_phase_residual_clip = should_run_final_fit_phase_residual_clip( + exotic_infoDict.get( + 'run_final_fit_phase_residual_clip', + exotic_infoDict.get( + 'Run Final-Fit Phase Residual Clip? (y/n)', + FINAL_FIT_PHASE_RESIDUAL_CLIP_DEFAULT, + ), + ) + ) + stellar_variability_only = should_run_stellar_variability_only( + exotic_infoDict.get('stellar_variability_only', STELLAR_VARIABILITY_ONLY_DEFAULT) + ) + require_apparent_magnitudes = should_require_apparent_magnitudes( + exotic_infoDict.get( + 'require_apparent_magnitudes', + REQUIRE_APPARENT_MAGNITUDES_DEFAULT, + ) + ) + use_exactly_the_comps_provided = should_use_exactly_the_comps_provided( + exotic_infoDict.get( + 'use_exactly_the_comps_provided', + USE_EXACTLY_PROVIDED_COMPARISONS_DEFAULT, + ) + ) + use_ensemble_photometry_for_stellar_variability = ( + should_use_ensemble_photometry_for_stellar_variability( + exotic_infoDict.get( + 'use_ensemble_photometry_for_stellar_variability', + STELLAR_VARIABILITY_ENSEMBLE_DEFAULT, + ) + ) + ) + use_ensemble_photometry_rather_than_single_comp = ( + should_use_ensemble_photometry_rather_than_single_comp( + exotic_infoDict.get('use_ensemble_photometry_rather_than_single_comp', 'n') + ) + ) + maximum_number_of_ensemble_comparisons_for_transit = ( + parse_maximum_number_of_ensemble_comparisons_for_transit( + exotic_infoDict.get( + 'maximum_number_of_ensemble_comparisons_for_transit', + TRANSIT_ENSEMBLE_MAX_COMPARISONS_DEFAULT, + ) + ) + ) + maximum_number_of_ensemble_comparisons_for_stellar_variability = ( + parse_maximum_number_of_ensemble_comparisons_for_stellar_variability( + exotic_infoDict.get( + 'maximum_number_of_ensemble_comparisons_for_stellar_variability', + STELLAR_VARIABILITY_ENSEMBLE_MAX_MEMBERS, + ) + ) + ) + provided_comparison_radec = [ + list(coords) for coords in (exotic_infoDict.get('comp_stars_radec') or []) + ] + comparisons_supplied_as_radec = bool(provided_comparison_radec) + provided_comparison_pixels = ( + [] + if comparisons_supplied_as_radec + else [list(coords) for coords in (exotic_infoDict.get('comp_stars') or [])] + ) + provided_comparison_count = ( + len(provided_comparison_radec) + if comparisons_supplied_as_radec + else len(exotic_infoDict.get('comp_stars') or []) + ) + if use_exactly_the_comps_provided: + if provided_comparison_count == 0 and fitsortext == 1: + log_info( + "Error: 'use_exactly_the_comps_provided' is enabled, but no comparison " + "coordinates were supplied.", + error=True, + ) + return + fixed_ensemble = provided_comparison_count > 1 + use_ensemble_photometry_rather_than_single_comp = fixed_ensemble + use_ensemble_photometry_for_stellar_variability = fixed_ensemble + if fixed_ensemble: + maximum_number_of_ensemble_comparisons_for_transit = provided_comparison_count + maximum_number_of_ensemble_comparisons_for_stellar_variability = provided_comparison_count + photometer_fortuitous_variables = should_photometer_fortuitous_variables( + exotic_infoDict.get( + 'photometer_fortuitous_variables', + PHOTOMETER_FORTUITOUS_VARIABLES_DEFAULT, + ) + ) + use_single_comparison_for_fortuitous_variables = ( + should_use_single_comparison_for_fortuitous_variables( + exotic_infoDict.get( + 'use_single_comparison_for_fortuitous_variables', + USE_SINGLE_COMPARISON_FOR_FORTUITOUS_VARIABLES_DEFAULT, + ) + ) + ) + use_nextastro_vsx_cache_first = should_use_nextastro_vsx_cache_first( + exotic_infoDict.get( + 'use_nextastro_vsx_cache_first', + USE_NEXTASTRO_VSX_CACHE_FIRST_DEFAULT, + ) + ) + if stellar_variability_only: + use_eebls_tmid_initializer = False + pick_comparison_by_eebls_snr = False + run_fast_ultranest_before_final_run = False + run_final_residual_rejection = False + run_final_fit_phase_residual_clip = False + use_legacy_psf_flux_mode = should_use_legacy_psf_flux_mode( + exotic_infoDict.get( + 'use_legacy_psf_flux', + exotic_infoDict.get( + 'legacy_psf_flux_mode', + LEGACY_PSF_FLUX_MODE_DEFAULT, + ), + ) + ) + psf_seed_track_directory = psf_seed_track_directory_from_config( + exotic_infoDict.get( + 'psf_seed_track_directory', + exotic_infoDict.get( + 'legacy_psf_seed_track_directory', + None, + ), + ) + ) + ultranest_min_num_live_points = configure_ultranest_min_num_live_points( + exotic_infoDict.get( + 'ultranest_min_num_live_points', + ULTRANEST_MIN_NUM_LIVE_POINTS_DEFAULT, + ) + ) + rprs_search_bound_max = configure_rprs_search_bound_max( + exotic_infoDict.get( + 'rprs_search_bound_max', + RPRS_SEARCH_BOUND_MAX_DEFAULT, + ) + ) + restrict_rprs_range, restrict_rprs_percentage = configure_rprs_range_restriction( + exotic_infoDict.get( + 'restrict_rprs_range', + RPRS_RANGE_RESTRICTION_DEFAULT, + ), + exotic_infoDict.get( + 'restrict_rprs_range_percentage', + RPRS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT, + ), + ) + use_prior_rprs_fallback_on_pinned_posterior = configure_prior_rprs_fallback_on_pinned_posterior( + exotic_infoDict.get( + 'use_prior_rprs_when_posterior_pinned', + exotic_infoDict.get( + 'use_prior_Rp/Rs_when_posterior_pinned', + RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR_DEFAULT, + ), + ) + ) + restrict_ars_range, restrict_ars_percentage = configure_ars_range_restriction( + exotic_infoDict.get( + 'restrict_ars_range', + ARS_RANGE_RESTRICTION_DEFAULT, + ), + exotic_infoDict.get( + 'restrict_ars_range_percentage', + ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT, + ), + ) + log_info(f"UltraNest minimum live points: {ultranest_min_num_live_points}.") + log_info(f"Rp/R* maximum search bound: {rprs_search_bound_max:.3f}.") + if restrict_rprs_range: + log_info( + "Rp/R* prior-centered search restriction enabled: " + f"+/- {restrict_rprs_percentage:.1f}% around the input prior." + ) + else: + log_info("Rp/R* prior-centered search restriction disabled.") + if use_prior_rprs_fallback_on_pinned_posterior: + log_info( + "Rp/R* pinned-posterior prior fallback enabled: when the posterior remains " + "edge-pinned after Rp/R* retry handling, EXOTIC reruns with Rp/R* fixed to " + "the input prior and quotes a data-only Rp/R* uncertainty." + ) + else: + log_info("Rp/R* pinned-posterior prior fallback disabled per optional_info setting.") + if restrict_ars_range: + log_info( + "a/Rs initial prior-centered search range enabled: at least " + f"+/- {restrict_ars_percentage:.1f}% around the input prior, widened when " + "five times the quoted a/Rs uncertainty is larger." + ) + else: + log_info("a/Rs prior-centered search restriction disabled.") + if run_fast_ultranest_before_final_run: + log_info( + "Fast pre-final UltraNest enabled: comparison-candidate UltraNest search runs " + f"with at most {FAST_ULTRANEST_MAX_BINNED_POINTS} binned light-curve point(s) " + f"when more than {FAST_ULTRANEST_MIN_POINTS_TO_BIN} points are available." + ) + else: + log_info("Fast pre-final UltraNest disabled per optional_info setting.") + if run_final_residual_rejection: + log_info( + "Final residual rejection enabled: the selected final light-curve fit will reject " + f"residual outliers beyond {FINAL_RESIDUAL_REJECTION_SIGMA:.1f} sigma and rerun UltraNest." + ) + else: + log_info("Final residual rejection disabled per optional_info setting.") + if run_final_fit_phase_residual_clip: + log_info("Final-fit phase residual clipping enabled.") + else: + log_info("Final-fit phase residual clipping disabled per optional_info setting.") + if use_legacy_psf_flux_mode: + log_info( + "Legacy PSF flux mode enabled: PSF photometry flux rows will use the 4.3.1-style " + "Gaussian fit with weighted-center override." + ) + else: + log_info("Modern PSF flux mode enabled.") + if psf_seed_track_directory is not None: + log_info( + f"PSF flux seed-track directory requested: {psf_seed_track_directory}" + ) + use_sparse_posterior_live_point_retry = configure_sparse_posterior_live_point_retry( + exotic_infoDict.get( + 'use_sparse_posterior_live_point_retry', + SPARSE_POSTERIOR_LIVE_POINT_RETRY_ENABLED_DEFAULT, + ) + ) + if stellar_variability_only: + use_sparse_posterior_live_point_retry = False + log_info( + "Stellar-variability-only mode enabled: EXOTIC will run the normal photometry and " + "comparison-star selection, discard predicted ingress-to-egress points, and skip transit fitting." + ) + if use_sparse_posterior_live_point_retry: + if run_fast_ultranest_before_final_run: + log_info( + "Selected comparison-star live-point extension enabled: comparison candidates " + "are ranked with fast pre-final UltraNest fits, then the chosen final comparison " + "fit reruns on the full-resolution light curve with " + f"{SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT}x additional minimum live points." + ) + else: + log_info( + "Selected comparison-star live-point extension enabled: comparison candidates " + "are ranked at the configured UltraNest live-point count, then the chosen final " + f"comparison fit continues with {SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT}x " + "additional minimum live points using its retained final-pass bounds." + ) + log_ultranest_mpi_status() + + # Keep non-final reduction products separate from the primary results. + Path(Path(exotic_infoDict['save']) / "working_artifacts").mkdir(exist_ok=True) + + archive_planet_dict = None if not args.override: - nea_obj = NASAExoplanetArchive(planet=userpDict['pName']) + nea_obj = NASAExoplanetArchive( + planet=userpDict['pName'], + non_interactive=args.non_interactive_run, + ) userpDict['pName'], CandidatePlanetBool, pDict = nea_obj.planet_info() + if isinstance(pDict, dict): + archive_planet_dict = dict(pDict) else: pDict = userpDict CandidatePlanetBool = False + + if file_cmd_opt == 2: + if args.nasaexoarch: + pass + elif args.override: + try: + pDict['ra'], pDict['dec'] = radec_hours_to_degree( + pDict.get('ra'), + pDict.get('dec'), + non_interactive_run=args.non_interactive_run, + target_name=pDict.get('pName'), + ) + except ValueError as coordinate_error: + if not args.non_interactive_run: + raise + + try: + coordinate_nea_obj = NASAExoplanetArchive( + planet=pDict.get('pName'), + non_interactive=True, + ) + _, _, coordinate_pdict = coordinate_nea_obj.planet_info() + except Exception as archive_error: + raise ValueError( + f"Non-interactive run cancelled for target {pDict.get('pName')}: the " + f"initialization-file coordinates are invalid ({coordinate_error}), and the " + f"NASA Exoplanet Archive coordinate lookup failed ({archive_error})." + ) from archive_error + + archive_ra = coordinate_pdict.get('ra') if isinstance(coordinate_pdict, dict) else None + archive_dec = coordinate_pdict.get('dec') if isinstance(coordinate_pdict, dict) else None + if isinstance(coordinate_pdict, dict): + archive_planet_dict = dict(coordinate_pdict) + pDict['ra'], pDict['dec'] = radec_hours_to_degree( + pDict.get('ra'), + pDict.get('dec'), + non_interactive_run=True, + archive_ra=archive_ra, + archive_dec=archive_dec, + target_name=pDict.get('pName'), + ) + else: + diff = False + + archive_ra = pDict.get('ra') if isinstance(pDict, dict) else None + archive_dec = pDict.get('dec') if isinstance(pDict, dict) else None + userpDict['ra'], userpDict['dec'] = radec_hours_to_degree( + userpDict.get('ra'), + userpDict.get('dec'), + non_interactive_run=args.non_interactive_run, + archive_ra=archive_ra, + archive_dec=archive_dec, + target_name=userpDict.get('pName'), + ) + + if not CandidatePlanetBool: + diff = check_parameters(userpDict, pDict) + if diff: + pDict = get_planetary_parameters(CandidatePlanetBool, userpDict, pdict=pDict) + else: + pDict = userpDict + else: + pDict = get_planetary_parameters(CandidatePlanetBool, userpDict, pdict=pDict) + + def lookup_archive_ephemeris(): + lookup_name = ( + pDict.get('pName') + if isinstance(pDict, dict) + else userpDict.get('pName') + ) + _, archive_candidate, archive_parameters = NASAExoplanetArchive( + planet=lookup_name, + non_interactive=True, + ).planet_info() + if archive_candidate or not isinstance(archive_parameters, dict): + return None + return archive_parameters + + pDict = resolve_required_transit_ephemeris( + pDict, + archive_planet_dict=archive_planet_dict, + archive_lookup=lookup_archive_ephemeris if args.override else None, + target_name=( + pDict.get('pName') + if isinstance(pDict, dict) + else userpDict.get('pName') + ), + ) + + target_search_restriction_prior = build_search_restriction_prior_from_planet_dict(pDict) + if restrict_ars_range and is_toi_or_tic_target(target_search_restriction_prior): + effective_ars_percentage = ars_initial_range_percentage_for_prior( + target_search_restriction_prior + ) + effective_ars_retries = ars_posterior_retry_limit_for_prior( + target_search_restriction_prior, + ARS_POSTERIOR_MAX_RETRIES_DEFAULT, + ) + log_info( + "TOI/TIC a/Rs search policy enabled: the initial half-width is the larger of " + f"30% and five times the quoted uncertainty ({effective_ars_percentage:.1f}% for this target), " + "and edge-pinned posteriors may expand beyond that initial range with " + f"up to {effective_ars_retries} automatic a/Rs refit(s)." + ) + # Seed random number generator (for run to run consistency) if exotic_infoDict['random_seed']: log_info(f"Setting random number seed to {exotic_infoDict['random_seed']}") @@ -1966,27 +32607,6 @@ def main(): demosaic_out = exotic_infoDict['demosaic_out'] demosaic_mult = calculate_demosaic_mult(demosaic_out) - if file_cmd_opt == 2: - if args.nasaexoarch: - pass - elif args.override: - if type(pDict['ra']) and type(pDict['dec']) is str: - pDict['ra'], pDict['dec'] = radec_hours_to_degree(pDict['ra'], pDict['dec']) - else: - diff = False - - if type(userpDict['ra']) and type(userpDict['dec']) is str: - userpDict['ra'], userpDict['dec'] = radec_hours_to_degree(userpDict['ra'], userpDict['dec']) - - if not CandidatePlanetBool: - diff = check_parameters(userpDict, pDict) - if diff: - pDict = get_planetary_parameters(CandidatePlanetBool, userpDict, pdict=pDict) - else: - pDict = userpDict - else: - pDict = get_planetary_parameters(CandidatePlanetBool, userpDict, pdict=pDict) - # check for Nans + Zeros for k in pDict: if k == 'rprs' and (pDict[k] == 0 or np.isnan(pDict[k])): @@ -2008,6 +32628,7 @@ def main(): log_info("\n**************************" "\nStarting Reduction Process" "\n**************************\n") + reduction_stage_timer.checkpoint("Initialization, configuration, and calibration masters") ######################################### # FLUX DATA EXTRACTION AND MANIPULATION @@ -2017,9 +32638,14 @@ def main(): plateStatus.initializeFilenames(exotic_infoDict['images']) inputfiles = corruption_check(exotic_infoDict['images']) + early_ignore_header_wcs = should_ignore_header_wcs(exotic_infoDict.get('ignore_header_wcs')) + if not early_ignore_header_wcs and maybe_reinterpret_decimal_ra_hours_from_wcs(inputfiles, pDict): + userpDict['ra'] = pDict['ra'] + userpDict['dec'] = pDict['dec'] # time sort images - times, jd_times = [], [] - for file in inputfiles: + times, jd_times, header_exptimes = [], [], [] + log_info(f"Reading FITS timestamps and converting to BJD_TDB for {len(inputfiles)} frame(s).") + for file_index, file in enumerate(inputfiles): extension = 0 plateStatus.setCurrentFilename(file) header = fits.getheader(filename=file, ext=extension) @@ -2030,6 +32656,10 @@ def main(): times.append(obsTime) plateStatus.setObsTime(obsTime) jd_times.append(img_time_jd(header)) + header_exptimes.append(get_exp_time(header)) + completed = file_index + 1 + if completed == len(inputfiles) or completed % 25 == 0: + log_info(f"Timestamp conversion progress: {completed}/{len(inputfiles)}") extension = 0 plateStatus.setCurrentFilename(inputfiles[0]) @@ -2045,117 +32675,1474 @@ def main(): exotic_infoDict['second_obs'] += ",MOBS" else: exotic_infoDict['second_obs'] = "MOBS" - exotic_infoDict['filter'] = "MObs CV" + exotic_infoDict['filter'] = "CV" exotic_infoDict['elev'] = 1268 exotic_infoDict['lat'] = 31.675467 exotic_infoDict['long'] = -110.951376 exotic_infoDict['pixel_bin'] = "2x2" - ld, ld0, ld1, ld2, ld3 = get_ld_values(pDict, exotic_infoDict) + exotic_infoDict.setdefault('observed_filter', exotic_infoDict.get('filter')) + log_info("Calculating limb-darkening coefficients.") + ld, ld0, ld1, ld2, ld3 = get_ld_values( + pDict, + exotic_infoDict, + non_interactive_run=args.non_interactive_run, + ) + log_info("Limb-darkening coefficients ready.") + reduction_stage_timer.checkpoint("FITS validation, timestamp conversion, and limb darkening") # check for EPW_MD5 checksum if 'EPW_MD5' in header: epw_md5 = header['EPW_MD5'] - si = np.argsort(times) - times = np.array(times)[si] - jd_times = np.array(jd_times)[si] - inputfiles = np.array(inputfiles)[si] - + si = np.argsort(times) + times = np.array(times)[si] + jd_times = np.array(jd_times)[si] + header_exptimes = np.array(header_exptimes, dtype=float)[si] + inputfiles = np.array(inputfiles)[si] + precheck_inputfile_count = int(len(inputfiles)) + finite_plot_times = times[np.isfinite(times)] + full_plot_time_range = None + if finite_plot_times.size: + full_plot_time_range = (float(np.min(finite_plot_times)), float(np.max(finite_plot_times))) + ignore_header_wcs = should_ignore_header_wcs(exotic_infoDict.get('ignore_header_wcs')) + allow_pixel_alignment_fallback = should_allow_pixel_alignment_fallback( + exotic_infoDict.get('allow_pixel_alignment_fallback', True) + ) + pixel_alignment_enabled = bool(ignore_header_wcs or allow_pixel_alignment_fallback) + if ignore_header_wcs: + log_info("Pixel alignment enabled explicitly: header WCS will be ignored for manual alignment.") + elif allow_pixel_alignment_fallback: + log_info( + "WCS coverage-aware coordinate mode enabled: per-frame WCS is preferred when coverage is " + "consistent, with pixel alignment fallback available for incomplete-WCS datasets." + ) + else: + log_info( + "WCS-authoritative coordinate mode enabled: each frame's header WCS will supply star pixel " + "positions; Astroalign/pixel alignment fallback is disabled." + ) + bad_wcs_threshold_fraction = get_bad_wcs_threshold_fraction( + exotic_infoDict.get('bad_wcs_threshold_percent') + ) + pointing_rejection_sigma = get_pointing_rejection_sigma( + exotic_infoDict.get('pointing_rejection_sigma') + ) + detect_bad_pixels_before_photometry = should_detect_bad_pixels_before_photometry( + exotic_infoDict.get('detect_bad_pixels_before_photometry', 'n') + ) + multiprocess_bad_pixel_precheck = get_multiprocess_bad_pixel_precheck_processes( + exotic_infoDict.get('multiprocess_bad_pixel_precheck', 'n') + ) + inputfiles, wcs_keep_mask, dropped_wcs_files = filter_sparse_missing_wcs_frames( + inputfiles, + ignore_header_wcs=ignore_header_wcs, + max_missing_fraction=bad_wcs_threshold_fraction, + allow_pixel_alignment_fallback=allow_pixel_alignment_fallback, + ) + if dropped_wcs_files: + times = times[wcs_keep_mask] + jd_times = jd_times[wcs_keep_mask] + header_exptimes = header_exptimes[wcs_keep_mask] + plateStatus.initializeFilenames(list(inputfiles)) + if len(inputfiles) == 0: + log_info( + "Error: no input frame has celestial WCS and pixel alignment fallback is disabled.", + error=True, + ) + return + post_wcs_inputfile_count = int(len(inputfiles)) + target_wcs_precheck_inputfiles = np.array(inputfiles, copy=True) + target_wcs_reference_file = inputfiles[0] if len(inputfiles) else None + inputfiles, target_wcs_keep_mask, dropped_target_wcs_files = filter_wcs_target_out_of_frame_frames( + inputfiles, + pDict, + obs_times=jd_times, + ignore_header_wcs=ignore_header_wcs, + ) + if dropped_target_wcs_files: + target_reference_fallback = reference_frame_rejection_fallback_info( + target_wcs_reference_file, + dropped_target_wcs_files, + ordered_inputfiles=target_wcs_precheck_inputfiles, + rejection_label="Target WCS precheck", + ) + times = times[target_wcs_keep_mask] + jd_times = jd_times[target_wcs_keep_mask] + header_exptimes = header_exptimes[target_wcs_keep_mask] + finite_plot_times = times[np.isfinite(times)] + full_plot_time_range = None + if finite_plot_times.size: + full_plot_time_range = (float(np.min(finite_plot_times)), float(np.max(finite_plot_times))) + plateStatus.initializeFilenames(list(inputfiles)) + else: + target_reference_fallback = None + if len(inputfiles) == 0: + log_info( + "Error: target WCS precheck removed every frame because the target RA/Dec projects outside " + "each image.", + error=True, + ) + return + post_target_wcs_inputfile_count = int(len(inputfiles)) + pointing_precheck_inputfiles = np.array(inputfiles, copy=True) + pointing_reference_file = inputfiles[0] if len(inputfiles) else None + inputfiles, pointing_keep_mask, dropped_pointing_files, pointing_alignment_transforms = filter_pointing_outlier_frames( + inputfiles, + pointing_rejection_sigma=pointing_rejection_sigma, + ignore_header_wcs=ignore_header_wcs, + allow_pixel_alignment_fallback=allow_pixel_alignment_fallback, + frame_loader=lambda file_name: load_calibrated_reduction_image( + file_name, + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + ), + return_alignment_transforms=True, + multiprocess_transformations=args.multiprocess_transformations, + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + demosaic_out=demosaic_out, + demosaic_mult=demosaic_mult, + ) + if dropped_pointing_files: + pointing_reference_fallback = reference_frame_rejection_fallback_info( + pointing_reference_file, + dropped_pointing_files, + ordered_inputfiles=pointing_precheck_inputfiles, + ) + reference_fallback = pointing_reference_fallback or target_reference_fallback + times = times[pointing_keep_mask] + jd_times = jd_times[pointing_keep_mask] + header_exptimes = header_exptimes[pointing_keep_mask] + finite_plot_times = times[np.isfinite(times)] + full_plot_time_range = None + if finite_plot_times.size: + full_plot_time_range = (float(np.min(finite_plot_times)), float(np.max(finite_plot_times))) + plateStatus.initializeFilenames(list(inputfiles)) + else: + reference_fallback = target_reference_fallback + if reference_fallback is not None and reference_fallback.get('next_reference_candidate') is None: + log_info( + "Error: all leading reference candidates were rejected by the pointing precheck; no usable " + "reference image remains.", + error=True, + ) + return + if reference_fallback is not None: + pointing_alignment_transforms = {} + post_pointing_inputfile_count = int(len(inputfiles)) + + bad_pixel_reference = None + if detect_bad_pixels_before_photometry: + log_info( + "Bad-pixel precheck enabled: scanning calibrated frames for persistent isolated " + "high-count outliers before plate-solve checks and photometry." + ) + bad_pixel_reference = build_persistent_bad_pixel_map( + inputfiles, + lambda file_name: load_calibrated_reduction_image( + file_name, + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + ), + save_directory=exotic_infoDict['save'], + max_processes=multiprocess_bad_pixel_precheck, + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + demosaic_out=demosaic_out, + demosaic_mult=demosaic_mult, + ) + else: + log_info("Bad-pixel precheck disabled per optional_info setting.") + exotic_UIprevTPX = exotic_infoDict['tar_coords'][0] exotic_UIprevTPY = exotic_infoDict['tar_coords'][1] # fit target in the first image and use it to determine aperture and annulus range inc = 0 - for ifile in inputfiles: - plateStatus.setCurrentFilename(ifile) - first_image = fits.getdata(ifile) - try: - initial_centroid = fit_centroid(first_image, [exotic_UIprevTPX, exotic_UIprevTPY], 0) - if np.isnan(initial_centroid[0]): - inc += 1 + if pixel_alignment_enabled and reference_fallback is None: + for ifile in inputfiles: + plateStatus.setCurrentFilename(ifile) + if bad_pixel_reference is not None: + first_image = load_calibrated_reduction_image( + ifile, + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + ) else: - break - except Exception: - inc += 1 - finally: - del first_image + first_image = fits.getdata(ifile) + try: + initial_centroid = fit_centroid(first_image, [exotic_UIprevTPX, exotic_UIprevTPY], 0) + if np.isnan(initial_centroid[0]): + inc += 1 + else: + break + except Exception: + inc += 1 + finally: + del first_image + elif reference_fallback is not None: + log_info( + "Skipping the old-pixel target precheck because the original reference image was " + "removed; the target will be projected from RA/Dec after the new reference WCS is ready.", + warn=True, + ) + else: + log.debug( + "Skipping the old-pixel target precheck in WCS-authoritative mode; target coordinates " + "will be projected independently from each frame's header WCS." + ) if inc > 0: log_info(f"Skipping first {inc} files - Target star not found") inputfiles = inputfiles[inc:] times = times[inc:] jd_times = jd_times[inc:] + header_exptimes = header_exptimes[inc:] + pointing_alignment_transforms = {} plateStatus.setCurrentFilename(inputfiles[0]) + header = get_first_image_header(inputfiles[0]) + + # For astrometry hints, prioritize coordinates explicitly provided by the user + # (from inits.json / CLI) over values scraped from NASA Exoplanet Archive. + hint_ra = userpDict.get('ra', pDict.get('ra')) + hint_dec = userpDict.get('dec', pDict.get('dec')) - wcs_file = check_wcs(inputfiles[0], exotic_infoDict['save'], exotic_infoDict['plate_opt']) + wcs_file = check_wcs(inputfiles[0], exotic_infoDict['save'], exotic_infoDict['plate_opt'], + use_nextastro_astrometry=args.use_nextastro_astrometry, + ra=hint_ra, dec=hint_dec, pixel_scale=exotic_infoDict.get('pixel_scale'), + ignore_header_wcs=ignore_header_wcs) img_scale_str, img_scale = get_img_scale(header, wcs_file, exotic_infoDict['pixel_scale']) + photometry_catalog_match_radius_arcsec = nextastro_catalog_match_radius_arcsec(img_scale) + log_info( + "NextAstro photometry matches for image-derived positions will use a " + f"{photometry_catalog_match_radius_arcsec:.2f} arcsec radius " + f"(max of {NEXTASTRO_PHOTOMETRY_MATCH_RADIUS_ARCSEC:.1f} arcsec and one image pixel)." + ) plateStatus.initializeComparisonStarCount(len(exotic_infoDict['comp_stars'])) ra_dec_tar, ra_dec_wcs = None, [] - chart_id, vsp_comp_stars, vsp_list = None, None, [] + chart_id, vsp_comp_stars, vsp_list = None, {}, [] + aavso_vsp_query_failed = False + nextastro_field_catalog = None + primary_target_catalog_match = None + science_comp_stars = [] + fortuitous_ensemble_stars = [] + fortuitous_auto_stars = [] + fortuitous_auto_scan_performed = False + fortuitous_variables = [] + fortuitous_calibration_stars = {} if wcs_file: - log_info(f"\nHere is the path to your plate solution: {wcs_file}") - wcs_header = fits.getheader(filename=wcs_file) - ra_wcs, dec_wcs = get_ra_dec(wcs_header) + if should_log_plate_solution_path(wcs_file): + log_info(f"\n{format_plate_solution_reference(wcs_file)}") + reference_image = fits.getdata(inputfiles[0]) + wcs_header = get_first_image_header(wcs_file) + ra_wcs, dec_wcs = get_ra_dec(wcs_header, image_shape=reference_image.shape) - exotic_UIprevTPX, exotic_UIprevTPY = check_target_pixel_wcs(exotic_UIprevTPX, exotic_UIprevTPY, - pDict, ra_wcs, dec_wcs, - fits.getdata(inputfiles[0]), - jd_times[0]) - ra_dec_tar = (ra_wcs[int(exotic_UIprevTPY)][int(exotic_UIprevTPX)], - dec_wcs[int(exotic_UIprevTPY)][int(exotic_UIprevTPX)]) + if comparisons_supplied_as_radec: + try: + exotic_infoDict['comp_stars'] = project_comparison_radec_to_pixels( + provided_comparison_radec, + wcs_header, + reference_image.shape, + ) + except ValueError as exc: + log_info( + "Error: supplied comparison-star RA/Dec coordinates are unusable: " + f"{exc}.", + error=True, + ) + return + provided_comparison_pixels = [ + list(coords) for coords in exotic_infoDict['comp_stars'] + ] + plateStatus.initializeComparisonStarCount(len(exotic_infoDict['comp_stars'])) + log_info( + f"Projected {len(exotic_infoDict['comp_stars'])} supplied comparison-star " + "RA/Dec coordinate(s) onto the selected reference image." + ) + + if reference_fallback is not None: + target_projection = estimate_target_pixel_from_ra_dec( + pDict, + wcs_header, + reference_image, + jd_times[0], + ) + if target_projection is None: + log_info( + "Error: could not estimate target coordinates from RA/Dec after removing the " + "rejected reference image.", + error=True, + ) + return + exotic_UIprevTPX, exotic_UIprevTPY, target_ra, target_dec = target_projection + exotic_infoDict['tar_coords'] = [exotic_UIprevTPX, exotic_UIprevTPY] + ra_dec_tar = (target_ra, target_dec) + else: + prefer_input_target_pixels = exotic_infoDict.get( + 'prefer_pixel_values_over_wcs_for_target', 'n' + ) + exotic_UIprevTPX, exotic_UIprevTPY = check_target_pixel_wcs( + exotic_UIprevTPX, + exotic_UIprevTPY, + pDict, + ra_wcs, + dec_wcs, + reference_image, + jd_times[0], + non_interactive_run=args.non_interactive_run, + wcs_header=wcs_header, + prefer_pixel_values_over_wcs_for_target=prefer_input_target_pixels, + ) + ra_dec_tar = (ra_wcs[int(exotic_UIprevTPY)][int(exotic_UIprevTPX)], + dec_wcs[int(exotic_UIprevTPY)][int(exotic_UIprevTPX)]) auid = vsx_auid(ra_dec_tar[0], ra_dec_tar[1]) - check_for_variable_stars(ra_wcs, dec_wcs, exotic_infoDict['comp_stars']) + if reference_fallback is not None and not comparisons_supplied_as_radec: + if use_exactly_the_comps_provided: + log_info( + "Error: the reference image containing the exact supplied comparison-star " + "pixels was rejected. EXOTIC will not replace or reproject those comparisons " + "while 'use_exactly_the_comps_provided' is enabled.", + error=True, + ) + return + old_comp_count = len(exotic_infoDict['comp_stars']) + exotic_infoDict['comp_stars'] = [] + log_info( + f"Reference fallback discarded {old_comp_count} supplied comparison-star pixel " + "coordinate(s) because they were tied to the rejected reference image.", + warn=True, + ) + + if exotic_infoDict['aavso_comp'] == 'y' and reference_fallback is None: + try: + vsp_comp_stars, chart_id = vsp_query( + wcs_file, + [header['NAXIS1'], header['NAXIS2']], + exotic_infoDict['filter'], + img_scale, + user_comp_stars=exotic_infoDict['comp_stars'], + user_targ_star=[exotic_UIprevTPX, exotic_UIprevTPY], + max_new_comp_stars=(0 if use_exactly_the_comps_provided else 2), + ) + vsp_list = [vsp_star['pos'] for vsp_star in vsp_comp_stars.values()] + except AAVSOVSPUnavailableError as exc: + aavso_vsp_query_failed = True + log_info( + "\nWarning: AAVSO VSP comparison-star lookup remains unavailable " + f"after five retries ({describe_retry_exception(exc)}). Continuing " + "without VSP data so the existing NextAstro/catalog fallback can run.", + warn=True, + ) - if exotic_infoDict['aavso_comp'] == 'y': - vsp_comp_stars, chart_id = vsp_query(wcs_file,[header['NAXIS1'], header['NAXIS2']], - exotic_infoDict['filter'], img_scale, - user_comp_stars=exotic_infoDict['comp_stars'], - user_targ_star = [ exotic_UIprevTPX, exotic_UIprevTPY ]) - vsp_list = [vsp_star['pos'] for vsp_star in vsp_comp_stars.values()] + try: + nextastro_field_catalog = nextastro_photometry_catalog_for_wcs( + wcs_file, + [header['NAXIS1'], header['NAXIS2']], + img_scale, + exotic_infoDict['filter'], + ) + except Exception as exc: + log_info( + "\nWarning: NextAstro full-field photometry catalog lookup failed " + f"({describe_retry_exception(exc)}). Will try a batched comparison-star lookup.", + warn=True, + ) + if nextastro_field_catalog is not None and ra_dec_tar is not None: + primary_target_catalog_match = nextastro_photometry_catalog_match( + nextastro_field_catalog, + ra_dec_tar[0], + ra_dec_tar[1], + exotic_infoDict['filter'], + max_separation_arcsec=photometry_catalog_match_radius_arcsec, + ) + + if reference_fallback is not None: + fallback_comp_stars, fallback_candidates = select_reference_fallback_comparison_stars( + reference_image, + reference_image.shape, + target_pixel=[exotic_UIprevTPX, exotic_UIprevTPY], + ) + log_reference_fallback_comparison_candidates( + fallback_comp_stars, + fallback_candidates, + ) + if fallback_comp_stars: + exotic_infoDict['comp_stars'] = fallback_comp_stars + else: + log_info( + "Error: no replacement image-detected comparison stars were available after " + "removing the rejected reference image.", + error=True, + ) + return + + automatic_calibration_selector_enabled = should_use_automatic_optimal_calibration_selector( + exotic_infoDict.get('automatic_optimal_calibration_selector', 'n') + ) + stellar_variability_ensemble_candidate_search = ( + stellar_variability_only + and use_ensemble_photometry_for_stellar_variability + ) + if ( + not use_exactly_the_comps_provided + and (automatic_calibration_selector_enabled or stellar_variability_ensemble_candidate_search) + ): + automatic_comp_count = parse_automatic_calibration_selector_count( + exotic_infoDict.get('automatic_optimal_calibration_selector_count') + ) + if stellar_variability_ensemble_candidate_search: + automatic_comp_count = max( + automatic_comp_count, + maximum_number_of_ensemble_comparisons_for_stellar_variability, + ) + elif ( + automatic_calibration_selector_enabled + and use_ensemble_photometry_rather_than_single_comp + ): + automatic_comp_count = max( + automatic_comp_count, + maximum_number_of_ensemble_comparisons_for_transit, + ) + ensemble_candidate_saturation_threshold = None + if stellar_variability_ensemble_candidate_search: + configured_candidate_saturation = parse_saturation_value( + exotic_infoDict.get( + 'saturation_value', + exotic_infoDict.get('saturation_value_adu', SATURATION_VALUE_DEFAULT), + ) + ) + header_candidate_saturation = saturation_value_from_header(header) + candidate_saturation = configured_candidate_saturation + if ( + header_candidate_saturation is not None + and configured_candidate_saturation == SATURATION_VALUE_DEFAULT + ): + candidate_saturation = header_candidate_saturation + ensemble_candidate_saturation_threshold = ( + candidate_saturation + * parse_overexposure_threshold_fraction( + exotic_infoDict.get( + 'overexposure_threshold_fraction', + OVEREXPOSURE_THRESHOLD_FRACTION_DEFAULT, + ) + ) + ) + automatic_comp_stars, automatic_candidates = select_automatic_optimal_calibration_stars( + reference_image, + reference_image.shape, + target_pixel=[exotic_UIprevTPX, exotic_UIprevTPY], + ra_wcs=ra_wcs, + dec_wcs=dec_wcs, + obs_filter=exotic_infoDict['filter'], + field_catalog=nextastro_field_catalog, + count=automatic_comp_count, + colour_term_metadata=colour_term_metadata_from_info(exotic_infoDict), + brightest_first=stellar_variability_ensemble_candidate_search, + saturation_threshold=ensemble_candidate_saturation_threshold, + catalog_match_radius_arcsec=photometry_catalog_match_radius_arcsec, + ) + log_automatic_optimal_calibration_selection( + automatic_comp_stars, + automatic_candidates, + automatic_comp_count, + brightest_first=stellar_variability_ensemble_candidate_search, + ) + if automatic_comp_stars: + exotic_infoDict['comp_stars'] = automatic_comp_stars + vsp_comp_stars = {} + + if not use_exactly_the_comps_provided: + check_for_variable_stars(ra_wcs, dec_wcs, exotic_infoDict['comp_stars'], + use_nextastro_variability_server=args.use_nextastro_variability_server) while not exotic_infoDict['comp_stars']: log_info("\nThere are no comparison stars left as all of them were indicated as variable stars." "\nPlease reenter new comparison star coordinates.") exotic_infoDict['comp_stars'] = comparison_star_coords(exotic_infoDict['comp_stars'], False) - check_for_variable_stars(ra_wcs, dec_wcs, exotic_infoDict['comp_stars']) - # Build RA/Dec for comp after list is finalized (avoid off by one issues, etc + check_for_variable_stars(ra_wcs, dec_wcs, exotic_infoDict['comp_stars'], + use_nextastro_variability_server=args.use_nextastro_variability_server) + + if use_exactly_the_comps_provided: + duplicate_comp_messages = [] + else: + exotic_infoDict['comp_stars'], duplicate_comp_messages = deduplicate_comparison_star_coords( + exotic_infoDict['comp_stars'] + ) + for duplicate_message in duplicate_comp_messages: + log_info(duplicate_message) + + science_comp_stars = [list(position) for position in exotic_infoDict['comp_stars']] + fortuitous_ensemble_stars = list(science_comp_stars) + configured_fortuitous_saturation = parse_saturation_value( + exotic_infoDict.get( + 'saturation_value', + exotic_infoDict.get('saturation_value_adu', SATURATION_VALUE_DEFAULT), + ) + ) + header_fortuitous_saturation = saturation_value_from_header(header) + fortuitous_saturation = configured_fortuitous_saturation + if ( + header_fortuitous_saturation is not None + and configured_fortuitous_saturation == SATURATION_VALUE_DEFAULT + ): + fortuitous_saturation = header_fortuitous_saturation + fortuitous_saturation_threshold = ( + fortuitous_saturation + * parse_overexposure_threshold_fraction( + exotic_infoDict.get( + 'overexposure_threshold_fraction', + OVEREXPOSURE_THRESHOLD_FRACTION_DEFAULT, + ) + ) + ) + + if photometer_fortuitous_variables: + if use_single_comparison_for_fortuitous_variables: + log_info( + "Fortuitous-variable single-comparison mode enabled (default): each " + "retained VSX target will use one unsaturated, non-variable, " + "catalog-calibrated comparison star." + ) + else: + log_info( + "Fortuitous-variable calibrated ensemble mode enabled per optional_info setting; " + "up to " + f"{maximum_number_of_ensemble_comparisons_for_stellar_variability} " + "comparisons will be used." + ) + fortuitous_variables = discover_fortuitous_vsx_variables( + wcs_file, + reference_image.shape, + img_scale, + reference_image, + exotic_infoDict['filter'], + target_pixel=[exotic_UIprevTPX, exotic_UIprevTPY], + field_catalog=nextastro_field_catalog, + exposure_seconds=(header_exptimes[0] if len(header_exptimes) else 1.0), + gain_e_per_adu=exotic_infoDict.get('gain_electrons_per_adu'), + saturation_threshold=fortuitous_saturation_threshold, + use_nextastro_vsx_cache_first=use_nextastro_vsx_cache_first, + ) + else: + log_info("Fortuitous-variable photometry disabled per optional_info setting.") + + if fortuitous_variables: + if use_exactly_the_comps_provided: + variable_comparison_rejections = [] + else: + science_comp_stars, variable_comparison_rejections = ( + filter_comparison_stars_against_fortuitous_variables( + science_comp_stars, + fortuitous_variables, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) + ) + if variable_comparison_rejections: + for rejection in variable_comparison_rejections: + log_info( + "Removed science comparison star " + f"#{rejection['comparison_index'] + 1} at " + f"[{rejection['position'][0]:.1f}, {rejection['position'][1]:.1f}] because " + f"the full-field VSX search identified the same source as " + f"{rejection['variable_name']} " + f"({rejection['distance_pixels']:.2f} pixel separation).", + warn=True, + ) + exotic_infoDict['comp_stars'] = [ + list(position) for position in science_comp_stars + ] + vsp_comp_stars = { + key: star + for key, star in vsp_comp_stars.items() + if fortuitous_variable_overlap( + star.get('pos'), + fortuitous_variables, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) is None + } + + if use_exactly_the_comps_provided: + fortuitous_auto_stars = [] + fortuitous_auto_scan_performed = True + log_info( + "Exact supplied-comparison mode: fortuitous-variable photometry will reuse " + "only the supplied comparison stars; skipping automatic comparison-pool " + "expansion." + ) + elif stellar_variability_ensemble_candidate_search: + # The stellar-variability target path just selected and VSX-vetted the + # same brightest-first pool with the same count and saturation limit. + # Reuse it rather than performing an identical full-field image scan. + fortuitous_auto_stars = [] + fortuitous_auto_scan_performed = True + log_info( + "Reusing the stellar-variability target comparison pool for fortuitous " + "VSX targets; skipping a duplicate automatic source scan." + ) + else: + fortuitous_auto_scan_performed = True + fortuitous_comp_count = parse_automatic_calibration_selector_count( + exotic_infoDict.get('automatic_optimal_calibration_selector_count') + ) + if not use_single_comparison_for_fortuitous_variables: + fortuitous_comp_count = max( + fortuitous_comp_count, + maximum_number_of_ensemble_comparisons_for_stellar_variability, + ) + fortuitous_auto_stars, _ = select_automatic_optimal_calibration_stars( + reference_image, + reference_image.shape, + target_pixel=[exotic_UIprevTPX, exotic_UIprevTPY], + ra_wcs=ra_wcs, + dec_wcs=dec_wcs, + obs_filter=exotic_infoDict['filter'], + field_catalog=nextastro_field_catalog, + count=fortuitous_comp_count, + colour_term_metadata=colour_term_metadata_from_info(exotic_infoDict), + brightest_first=True, + saturation_threshold=fortuitous_saturation_threshold, + catalog_match_radius_arcsec=photometry_catalog_match_radius_arcsec, + ) + check_for_variable_stars( + ra_wcs, + dec_wcs, + fortuitous_auto_stars, + use_nextastro_variability_server=args.use_nextastro_variability_server, + ) + fortuitous_auto_stars, automatic_variable_rejections = ( + filter_comparison_stars_against_fortuitous_variables( + fortuitous_auto_stars, + fortuitous_variables, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) + ) + for rejection in automatic_variable_rejections: + log_info( + "Removed automatic fortuitous-variable comparison candidate at " + f"[{rejection['position'][0]:.1f}, {rejection['position'][1]:.1f}] because " + f"the full-field VSX search identified {rejection['variable_name']} " + f"at the same source ({rejection['distance_pixels']:.2f} pixel separation).", + warn=True, + ) + fortuitous_ensemble_stars, fortuitous_duplicate_messages = ( + build_tracked_comparison_pool( + science_comp_stars, + fortuitous_auto_stars, + use_exactly_the_comps_provided=use_exactly_the_comps_provided, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) + ) + for duplicate_message in fortuitous_duplicate_messages: + log_info(duplicate_message) + log_info( + "Fortuitous-variable pool contains " + f"{len(fortuitous_ensemble_stars)} non-variable, non-saturated, " + "catalog-matched comparison candidate(s)." + ) + + ensemble_ra_dec = build_comp_ra_dec( + ra_wcs, + dec_wcs, + fortuitous_ensemble_stars, + ) + vsp_comp_stars = merge_nextastro_calibration_stars( + science_comp_stars, + ensemble_ra_dec[:len(science_comp_stars)], + exotic_infoDict['filter'], + existing_comp_stars=vsp_comp_stars, + field_catalog=nextastro_field_catalog, + match_radius_arcsec=photometry_catalog_match_radius_arcsec, + ) + usable_science_nextastro_v = any( + star.get('catalog_source') == 'NextAstro photometry catalog' + and catalog_calibration_is_usable_for_filter( + star, + exotic_infoDict['filter'], + max_error=CATALOG_REFERENCE_MAGNITUDE_ERROR_MAX, + ) + for star in vsp_comp_stars.values() + if isinstance(star, dict) + ) + if ( + str( + preferred_catalog_magnitude_band_for_filter(exotic_infoDict['filter']) or '' + ).upper() == 'V' + and not usable_science_nextastro_v + and not fortuitous_auto_scan_performed + and not use_exactly_the_comps_provided + ): + fortuitous_auto_scan_performed = True + fortuitous_comp_count = parse_automatic_calibration_selector_count( + exotic_infoDict.get('automatic_optimal_calibration_selector_count') + ) + if not use_single_comparison_for_fortuitous_variables: + fortuitous_comp_count = max( + fortuitous_comp_count, + maximum_number_of_ensemble_comparisons_for_stellar_variability, + ) + fortuitous_auto_stars, _ = select_automatic_optimal_calibration_stars( + reference_image, + reference_image.shape, + target_pixel=[exotic_UIprevTPX, exotic_UIprevTPY], + ra_wcs=ra_wcs, + dec_wcs=dec_wcs, + obs_filter=exotic_infoDict['filter'], + field_catalog=nextastro_field_catalog, + count=fortuitous_comp_count, + colour_term_metadata=colour_term_metadata_from_info(exotic_infoDict), + brightest_first=True, + saturation_threshold=fortuitous_saturation_threshold, + catalog_match_radius_arcsec=photometry_catalog_match_radius_arcsec, + ) + check_for_variable_stars( + ra_wcs, + dec_wcs, + fortuitous_auto_stars, + use_nextastro_variability_server=args.use_nextastro_variability_server, + ) + fortuitous_auto_stars, automatic_variable_rejections = ( + filter_comparison_stars_against_fortuitous_variables( + fortuitous_auto_stars, + fortuitous_variables, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) + ) + for rejection in automatic_variable_rejections: + log_info( + "Removed automatic NextAstro V calibration candidate at " + f"[{rejection['position'][0]:.1f}, {rejection['position'][1]:.1f}] because " + f"the full-field VSX search identified {rejection['variable_name']} at the " + f"same source ({rejection['distance_pixels']:.2f} pixel separation).", + warn=True, + ) + fortuitous_ensemble_stars, duplicate_messages = ( + build_tracked_comparison_pool( + science_comp_stars, + fortuitous_auto_stars, + use_exactly_the_comps_provided=use_exactly_the_comps_provided, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) + ) + for duplicate_message in duplicate_messages: + log_info(duplicate_message) + ensemble_ra_dec = build_comp_ra_dec( + ra_wcs, + dec_wcs, + fortuitous_ensemble_stars, + ) + log_info( + "Full-field NextAstro V calibration search expanded the tracked comparison " + f"pool to {len(fortuitous_ensemble_stars)} star(s) before considering AAVSO VSP." + ) + fortuitous_calibration_stars = merge_nextastro_calibration_stars( + fortuitous_ensemble_stars, + ensemble_ra_dec, + exotic_infoDict['filter'], + existing_comp_stars=vsp_comp_stars, + field_catalog=nextastro_field_catalog, + match_radius_arcsec=photometry_catalog_match_radius_arcsec, + ) + _, fallback_vsp_stars, fallback_chart_id, fallback_vsp_queried = ( + merge_aavso_vsp_v_calibration_fallback( + wcs_file, + [header['NAXIS1'], header['NAXIS2']], + exotic_infoDict['filter'], + img_scale, + fortuitous_calibration_stars, + science_comp_stars, + user_targ_star=[exotic_UIprevTPX, exotic_UIprevTPY], + max_new_comp_stars= + ( + 0 + if use_exactly_the_comps_provided + else maximum_number_of_ensemble_comparisons_for_stellar_variability + ), + vsp_query_available=not aavso_vsp_query_failed, + ) + ) + if fallback_chart_id is not None: + chart_id = fallback_chart_id + if fallback_vsp_queried and fallback_vsp_stars: + fallback_vsp_stars = { + label: star + for label, star in fallback_vsp_stars.items() + if fortuitous_variable_overlap( + star.get('pos'), + fortuitous_variables, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) is None + } + if use_exactly_the_comps_provided: + fallback_variable_rejections = [] + else: + science_comp_stars, fallback_variable_rejections = ( + filter_comparison_stars_against_fortuitous_variables( + science_comp_stars, + fortuitous_variables, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) + ) + for rejection in fallback_variable_rejections: + log_info( + "Removed AAVSO VSP comparison star at " + f"[{rejection['position'][0]:.1f}, {rejection['position'][1]:.1f}] because " + f"the full-field VSX search identified {rejection['variable_name']} at the " + f"same source ({rejection['distance_pixels']:.2f} pixel separation).", + warn=True, + ) + vsp_comp_stars.update(fallback_vsp_stars) + fortuitous_ensemble_stars, fallback_duplicate_messages = ( + merge_automatic_comparison_star_coords( + science_comp_stars, + fortuitous_auto_stars, + duplicate_radius_pixels=REFERENCE_FALLBACK_DEDUPE_RADIUS_PIXELS, + ) + ) + for duplicate_message in fallback_duplicate_messages: + log_info(duplicate_message) + ensemble_ra_dec = build_comp_ra_dec( + ra_wcs, + dec_wcs, + fortuitous_ensemble_stars, + ) + vsp_comp_stars = merge_nextastro_calibration_stars( + science_comp_stars, + ensemble_ra_dec[:len(science_comp_stars)], + exotic_infoDict['filter'], + existing_comp_stars=vsp_comp_stars, + field_catalog=nextastro_field_catalog, + match_radius_arcsec=photometry_catalog_match_radius_arcsec, + ) + fortuitous_calibration_stars = merge_nextastro_calibration_stars( + fortuitous_ensemble_stars, + ensemble_ra_dec, + exotic_infoDict['filter'], + existing_comp_stars=vsp_comp_stars, + field_catalog=nextastro_field_catalog, + match_radius_arcsec=photometry_catalog_match_radius_arcsec, + ) + # The target variability plot can use any tracked, VSX-vetted + # catalog calibration, including full-field NextAstro candidates + # that were added for fortuitous-variable photometry. + vsp_comp_stars = dict(fortuitous_calibration_stars) + tracked_positions = [*fortuitous_ensemble_stars] + for variable in fortuitous_variables: + variable['tracking_key'] = f"comp{len(tracked_positions) + 1}" + tracked_positions.append(list(variable['pos'])) + exotic_infoDict['comp_stars'] = tracked_positions + # Build RA/Dec after the tracking list is finalized. Science comps remain first, + # followed by fortuitous-only ensemble candidates and then the VSX targets. ra_dec_wcs = build_comp_ra_dec(ra_wcs, dec_wcs, exotic_infoDict['comp_stars']) + if ( + comparisons_supplied_as_radec + and use_exactly_the_comps_provided + and len(ra_dec_wcs) >= len(provided_comparison_radec) + ): + # Preserve the user's celestial coordinates exactly for + # frame-to-frame WCS tracking and output metadata. The + # projected pixels remain the reference-frame centroids. + ra_dec_wcs[:len(provided_comparison_radec)] = [ + list(coords) for coords in provided_comparison_radec + ] + vsp_list = [vsp_star['pos'] for vsp_star in vsp_comp_stars.values()] + plateStatus.initializeComparisonStarCount(len(exotic_infoDict['comp_stars'])) + else: + if comparisons_supplied_as_radec: + log_info( + "Error: 'Comparison Star(s) RA & Dec' requires a usable celestial WCS on the " + "selected reference image so the supplied stars can be projected into pixels.", + error=True, + ) + return + if reference_fallback is not None: + log_info( + "Error: the original reference image was removed, but the new reference image does not " + "have a usable WCS. EXOTIC cannot estimate target coordinates from RA/Dec or choose " + "replacement image-detected comparison stars without a new reference WCS.", + error=True, + ) + return + if photometer_fortuitous_variables: + log_info( + "Warning: fortuitous-variable photometry requires a usable celestial WCS; " + "the full-field VSX search will be skipped for this reduction.", + warn=True, + ) + if use_exactly_the_comps_provided: + duplicate_comp_messages = [] + else: + exotic_infoDict['comp_stars'], duplicate_comp_messages = deduplicate_comparison_star_coords( + exotic_infoDict['comp_stars'] + ) + for duplicate_message in duplicate_comp_messages: + log_info(duplicate_message) + science_comp_stars = [list(position) for position in exotic_infoDict['comp_stars']] + fortuitous_ensemble_stars = list(science_comp_stars) plateStatus.initializeComparisonStarCount(len(exotic_infoDict['comp_stars'])) - - # aperture sizes in stdev (sigma) of PSF - apers = np.linspace(1.5, 6, 20) - annuli = np.linspace(6, 15, 19) # alloc psf fitting param psf_data = { # x-cent, y-cent, amplitude, sigma-x, sigma-y, rotation, offset 'target': np.zeros((len(inputfiles), 7)), # PSF fit } - aper_data = { - 'target': np.zeros((len(inputfiles), len(apers), len(annuli))), - 'target_bg': np.zeros((len(inputfiles), len(apers), len(annuli))) + psf_flux_data = { + 'target': np.zeros((len(inputfiles), 7)), } tar_comp_dist = {} vsp_num = [] + comp_star_count = len(exotic_infoDict['comp_stars']) + tracked_vsx_labels = { + str(variable.get('tracking_key')): f"Tracked VSX variable {variable.get('name', 'unknown')}" + for variable in fortuitous_variables + if variable.get('tracking_key') + } + plateStatus.setComparisonStarLabels({ + int(comp_key[4:]): label + for comp_key, label in tracked_vsx_labels.items() + if comp_key.startswith('comp') and comp_key[4:].isdigit() + }) + aperture_estimation_stars = aperture_estimation_comparison_stars( + science_comp_stars, + stellar_variability_only=stellar_variability_only, + ) + aperture_estimation_comp_count = len(aperture_estimation_stars) + aperture_estimation_comp_indices = tuple(range(aperture_estimation_comp_count)) + aperture_estimation_includes_target = not stellar_variability_only + aperture_frame_sigma_comp_indices = ( + aperture_estimation_comp_indices if stellar_variability_only else None + ) + psf_noise_data = initialize_psf_noise_data(len(inputfiles), comp_star_count) + target_overexposed_frame_mask = np.zeros(len(inputfiles), dtype=bool) + comp_overexposed_masks = { + f"comp{comp_idx + 1}": np.zeros(len(inputfiles), dtype=bool) + for comp_idx in range(comp_star_count) + } + frame_noise_configs = [] + require_comp_star = resolve_require_comp_star_for_exposure_times( + exotic_infoDict.get('require_comp_star', 'y'), + header_exptimes, + ) + target_driven_comp_selection = is_target_driven_comp_selection_enabled( + exotic_infoDict.get('target_driven_comp_selection', 'n') + ) + skip_low_comp_coverage_rejection = should_skip_low_comparison_coverage_rejection( + exotic_infoDict.get('skip_low_comparison_coverage_rejection', 'n') + ) + if skip_low_comp_coverage_rejection: + log_info("Skipping low-coverage comparison-star rejection per optional_info setting.") + fit_every_comparison_candidate = should_fit_lightcurve_to_every_comparison_candidate( + exotic_infoDict.get('fit_lightcurve_to_every_comparison_candidate', 'n') + ) + use_deviation_from_expected_transit_in_qc = should_use_deviation_from_expected_transit_in_qc( + exotic_infoDict.get('use_deviation_from_expected_transit_in_qc', True) + ) + deviation_from_expected_transit_in_qc_sigma = parse_deviation_from_expected_transit_in_qc_sigma( + exotic_infoDict.get('deviation_from_expected_transit_in_qc_sigma', 5.0) + ) + exit_at_first_qc_pass_solution = should_exit_at_first_qc_pass_solution( + exotic_infoDict.get('exit_at_first_qc_pass_solution', 'y') + ) + use_psf_photometry = should_use_psf_photometry( + exotic_infoDict.get('use_psf_photometry', 'y') + ) + use_aperture_photometry = should_use_aperture_photometry( + exotic_infoDict.get('use_aperture_photometry', 'y') + ) + use_adaptive_apertures = is_adaptive_aperture_mode_enabled( + exotic_infoDict.get('use_adaptive_apertures', False) + ) + use_aperture_corrections_and_full_image_fwhm = should_use_aperture_corrections_and_full_image_fwhm( + exotic_infoDict.get('use_aperture_corrections_and_full_image_fwhm', False) + ) + reject_overexposed_stars = should_reject_overexposed_stars( + exotic_infoDict.get('reject_overexposed_stars', REJECT_OVEREXPOSED_STARS_DEFAULT) + ) + if ( + ( + (stellar_variability_only and use_ensemble_photometry_for_stellar_variability) + or (photometer_fortuitous_variables and bool(fortuitous_variables)) + ) + and not reject_overexposed_stars + ): + reject_overexposed_stars = True + log_info( + "Stellar-variability ensemble membership requires non-saturated stars; " + "overexposure rejection will remain enabled for this run.", + warn=True, + ) + configured_saturation_value = parse_saturation_value( + exotic_infoDict.get( + 'saturation_value', + exotic_infoDict.get('saturation_value_adu', SATURATION_VALUE_DEFAULT), + ) + ) + header_saturation_value = saturation_value_from_header(header) + saturation_value = configured_saturation_value + if ( + header_saturation_value is not None + and configured_saturation_value == SATURATION_VALUE_DEFAULT + ): + saturation_value = header_saturation_value + exotic_infoDict['saturation_value'] = saturation_value + log_info( + f"Using FITS header-derived saturation_value={saturation_value:.1f} " + "for overexposure rejection." + ) + overexposure_threshold_fraction = parse_overexposure_threshold_fraction( + exotic_infoDict.get( + 'overexposure_threshold_fraction', + OVEREXPOSURE_THRESHOLD_FRACTION_DEFAULT, + ) + ) + overexposure_threshold = saturation_value * overexposure_threshold_fraction + if not use_psf_photometry and not use_aperture_photometry: + log_info("Error: both PSF and aperture photometry are disabled in optional_info.", error=True) + return + if not use_psf_photometry: + log_info("PSF photometry disabled per optional_info setting.") + if not use_aperture_photometry: + log_info("Aperture photometry disabled per optional_info setting.") + if not use_eebls_tmid_initializer: + log_info("EEBLS transit initializer disabled per optional_info setting.") + if not pick_comparison_by_eebls_snr: + log_info("Comparison-star selection by EEBLS SNR disabled per optional_info setting.") + if use_exactly_the_comps_provided: + exact_set_preserved = ( + len(science_comp_stars) == len(provided_comparison_pixels) + and np.allclose( + np.asarray(science_comp_stars, dtype=float), + np.asarray(provided_comparison_pixels, dtype=float), + rtol=0.0, + atol=1e-9, + ) + ) + if not exact_set_preserved: + log_info( + "Error: the supplied comparison-star set changed before photometry; " + "exact-comparison mode will not continue with an added, removed, or " + "substituted comparison.", + error=True, + ) + return + exact_mode = "single comparison" if len(science_comp_stars) == 1 else "fixed ensemble" + log_info( + "Exact supplied-comparison mode enabled: EXOTIC will use the " + f"{len(science_comp_stars)} supplied comparison coordinate(s) as a {exact_mode}, " + "without automatic replacement, VSX rejection, stability vetting, ranking, or " + "ensemble-size limiting." + ) + for comp_index, pixel_position in enumerate(science_comp_stars, start=1): + if comparisons_supplied_as_radec: + ra_deg, dec_deg = provided_comparison_radec[comp_index - 1] + log_info( + f" Exact supplied comp #{comp_index}: " + f"RA={ra_deg:.8f} deg, Dec={dec_deg:.8f} deg -> " + f"pixels=[{pixel_position[0]:.3f}, {pixel_position[1]:.3f}]" + ) + else: + log_info( + f" Exact supplied comp #{comp_index}: " + f"pixels=[{pixel_position[0]:.3f}, {pixel_position[1]:.3f}]" + ) + elif use_ensemble_photometry_rather_than_single_comp: + log_info( + "Ensemble comparison photometry enabled per optional_info setting; the final target " + "light curve will use non-rejected comparison stars as a combined reference, up to " + "maximum_number_of_ensemble_comparisons_for_transit=" + f"{maximum_number_of_ensemble_comparisons_for_transit}." + ) + if not require_apparent_magnitudes: + log_info( + "Apparent magnitudes are not required per optional_info; differential-magnitude " + "products will still be written, and any available apparent calibration remains optional." + ) + if stellar_variability_only and not use_exactly_the_comps_provided: + if use_ensemble_photometry_for_stellar_variability: + log_info( + "Stellar-variability calibrated ensemble enabled (default): EXOTIC will combine " + "bright, unsaturated, VSX-vetted comparison stars after clipping high catalog " + "magnitude uncertainties, up to " + "maximum_number_of_ensemble_comparisons_for_stellar_variability=" + f"{maximum_number_of_ensemble_comparisons_for_stellar_variability}." + ) + else: + log_info( + "Stellar-variability calibrated ensemble disabled per optional_info setting; " + "EXOTIC will select one comparison star by out-of-transit scatter." + ) + if reject_overexposed_stars: + log_info( + "Overexposed-star rejection enabled: target frames and comparison-star measurements " + f"with aperture pixels above {overexposure_threshold:.1f} will be rejected " + f"(saturation_value={saturation_value:.1f}, " + f"threshold_fraction={overexposure_threshold_fraction:.3f})." + ) + else: + log_info("Overexposed-star rejection disabled per optional_info setting.") + if not use_deviation_from_expected_transit_in_qc: + log_info("Expected-value transit QC deviation checks disabled per optional_info setting.") + if target_driven_comp_selection: + log_info( + "Warning: target-driven comparison selection is no longer used; " + "EXOTIC will run comparison-star calibration followed by full candidate reductions.", + warn=True, + ) + if not exit_at_first_qc_pass_solution: + log_info( + "Comparison-star candidate search will evaluate all ranked candidates before selection " + "because 'exit_at_first_qc_pass_solution' is disabled." + ) + + pDict['use_deviation_from_expected_transit_in_qc'] = use_deviation_from_expected_transit_in_qc + pDict['deviation_from_expected_transit_in_qc_sigma'] = deviation_from_expected_transit_in_qc_sigma for i, coord in enumerate(exotic_infoDict['comp_stars']): ckey = f"comp{i + 1}" if coord in vsp_list: vsp_num.append(i) psf_data[ckey] = np.zeros((len(inputfiles), 7)) - aper_data[ckey] = np.zeros((len(inputfiles), len(apers), len(annuli))) - aper_data[f"{ckey}_bg"] = np.zeros((len(inputfiles), len(apers), len(annuli))) + psf_flux_data[ckey] = np.zeros((len(inputfiles), 7)) tar_comp_dist[ckey] = np.zeros(2) + coarse_tune_frames = 0 + coarse_tune_frame_indices = np.array([], dtype=int) + coarse_apertures_sigma = None + coarse_annuli_sigma = None + if use_aperture_photometry: + coarse_tune_frames = min(len(inputfiles), APERTURE_AUTOTUNE_MAX_FRAMES) + if len(inputfiles) >= APERTURE_AUTOTUNE_MIN_FRAMES: + coarse_tune_frames = max(APERTURE_AUTOTUNE_MIN_FRAMES, coarse_tune_frames) + coarse_tune_frame_indices = evenly_spaced_aperture_tuning_indices( + len(inputfiles), + max_frames=coarse_tune_frames, + ) + coarse_apertures_sigma = np.linspace( + APERTURE_SIGMA_MIN, + APERTURE_SIGMA_MAX, + APERTURE_AUTOTUNE_COARSE_APER_POINTS, + ) + coarse_annuli_sigma = np.linspace( + ANNULUS_SIGMA_MIN, + ANNULUS_SIGMA_MAX, + APERTURE_AUTOTUNE_COARSE_ANNULUS_POINTS, + ) + log_info( + "Automatic aperture tuning enabled: " + f"coarse_grid={len(coarse_apertures_sigma)}x{len(coarse_annuli_sigma)}, " + f"coarse_frames={coarse_tune_frames}, sampling=evenly_spaced_full_sequence." + ) + + sigma = np.nan + coarse_aperture_values = None + coarse_annulus_values = None + aperture_values = None + annulus_values = None + apers = None + annuli = None + aperture_grid_tuned = False + coarse_aper_data = None + if use_aperture_photometry: + coarse_aper_data = initialize_aperture_data_store( + coarse_tune_frames, + len(coarse_apertures_sigma), + len(coarse_annuli_sigma), + comp_star_count, + ) + aper_data = None + coarse_frame_cache = [None] * coarse_tune_frames if use_aperture_photometry else [] + + target_and_comp_radec = None + if ra_dec_tar is not None and ra_dec_wcs: + target_and_comp_radec = np.array([ra_dec_tar, *ra_dec_wcs], dtype=float) + target_and_comp_pixels = np.array( + [[exotic_UIprevTPX, exotic_UIprevTPY], *exotic_infoDict['comp_stars']], + dtype=float, + ) + fast_aperture_mask = is_fast_aperture_mask_enabled(exotic_infoDict.get('fast_aperture_mask')) + if use_aperture_photometry and use_adaptive_apertures: + log_info( + "Adaptive aperture scaling enabled: evaluating aperture candidates in PSF sigma units per frame " + f"(1 image FWHM = {GAUSSIAN_SIGMA_TO_FWHM:.3f} sigma)." + ) + if use_aperture_photometry: + log_info( + "Aperture candidates are limited to " + f"{APERTURE_MIN_FWHM_MULTIPLIER:.1f}-{APERTURE_MAX_FWHM_MULTIPLIER:.1f} image FWHM " + f"({APERTURE_SIGMA_MIN:.2f}-{APERTURE_SIGMA_MAX:.2f} sigma)." + ) + if use_aperture_corrections_and_full_image_fwhm: + log_info("Aperture corrections and full-image FWHM estimation enabled per optional_info setting.") + if stellar_variability_only: + if use_ensemble_photometry_for_stellar_variability: + estimator_text = ( + f"{aperture_estimation_comp_count} bright, reference-frame non-saturated, " + "VSX-vetted non-variable comparison star(s)" + ) + else: + estimator_text = ( + f"the first {aperture_estimation_comp_count} supplied science comparison star(s), " + "with existing saturation and PSF-quality masks" + ) + log_info( + f"Stellar-variability aperture estimation will use only {estimator_text}. " + "The variable science target and additional tracked stars will be measured once " + "with the selected aperture." + ) + elif comp_star_count > aperture_estimation_comp_count: + log_info( + "Aperture-grid estimation will use only the science target and " + f"{aperture_estimation_comp_count} science comparison star(s); " + f"{comp_star_count - aperture_estimation_comp_count} fortuitous-only tracked " + "star(s) will reuse the selected aperture." + ) + + reduction_stage_timer.checkpoint("Frame prechecks, WCS, catalogs, and comparison preparation") + # open files, calibrate, align, photometry + reset_transform_timing_stats() + reset_photometry_timing_stats() + multiprocess_alignment_results = None + multiprocess_alignment_results_applied = False + aperture_preselected_from_sample = False + aperture_tuning_sample_score = np.nan + use_multiprocess_alignment = ( + args.multiprocess_transformations is not None and args.multiprocess_transformations > 0 + ) + comp_alignment_keys = [f"comp{j + 1}" for j in range(comp_star_count)] + psf_flux_seed_tracks = load_psf_flux_seed_tracks( + psf_seed_track_directory, + len(inputfiles), + comp_alignment_keys, + ) + if use_multiprocess_alignment: + multiprocess_alignment_results = build_multiprocess_alignment_results( + inputfiles, + args.multiprocess_transformations, + target_and_comp_pixels, + target_and_comp_radec=target_and_comp_radec, + ignore_header_wcs=ignore_header_wcs, + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + demosaic_out=demosaic_out, + demosaic_mult=demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + use_fast_centroid_cadence=False, + use_adaptive_apertures=use_adaptive_apertures, + compute_fallback_transform=pixel_alignment_enabled, + precomputed_fallback_transforms=pointing_alignment_transforms, + ) + for alignment_index, alignment_result in enumerate(multiprocess_alignment_results): + apply_parallel_alignment_result( + alignment_result, + alignment_index, + psf_data, + tar_comp_dist, + comp_alignment_keys, + ) + multiprocess_alignment_results_applied = True + + if ( + use_aperture_photometry + and multiprocess_alignment_results_applied + and aperture_estimation_comp_count > 0 + and not use_aperture_corrections_and_full_image_fwhm + ): + aperture_tuning_start = perf_counter() + sigma = aperture_frame_sigma_from_psf_data( + psf_data, + 0, + comparison_indices=aperture_frame_sigma_comp_indices, + ) + if not np.isfinite(sigma) or sigma <= 0: + sigma = 1.0 + + memmap_cutouts = can_memmap_aperture_tuning_cutouts( + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + bad_pixel_reference=bad_pixel_reference, + ) + log_info( + "Aperture tuning sample: " + f"{len(coarse_tune_frame_indices)} evenly spaced frame(s) spanning " + f"1-{len(inputfiles)}; image access=" + + ("FITS memmap star cutouts" if memmap_cutouts else "calibrated full-frame fallback") + + "." + ) + tuning_cutouts, tuning_airmass, tuning_overexposed_masks = build_aperture_tuning_cutouts( + inputfiles, + coarse_tune_frame_indices, + psf_data, + aperture_estimation_comp_indices, + use_adaptive_apertures, + sigma, + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + demosaic_out=demosaic_out, + demosaic_mult=demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + p_dict=pDict, + info_dict=exotic_infoDict, + jd_times=jd_times, + reject_overexposed=reject_overexposed_stars, + overexposure_threshold=overexposure_threshold, + fast_aperture_mask=fast_aperture_mask, + ) + tuning_psf_data = { + key: np.asarray(values)[coarse_tune_frame_indices] + for key, values in psf_data.items() + } + coarse_aper_data = populate_aperture_tuning_data_from_cutouts( + tuning_cutouts, + coarse_apertures_sigma if use_adaptive_apertures else coarse_apertures_sigma * sigma, + coarse_annuli_sigma if use_adaptive_apertures else coarse_annuli_sigma * sigma, + aperture_estimation_comp_indices, + use_adaptive_apertures, + sigma, + fast_aperture_mask=fast_aperture_mask, + ) + for tuning_frame_index in range(len(tuning_cutouts)): + apply_overexposure_masks_to_aperture_frame( + coarse_aper_data, + tuning_frame_index, + False, + tuning_overexposed_masks, + ) + tuning_psf_quality_masks = { + f"comp{comp_idx + 1}": psf_quality_mask_for_key( + tuning_psf_data, + f"comp{comp_idx + 1}", + len(tuning_cutouts), + ) + for comp_idx in range(aperture_estimation_comp_count) + } + refined_apertures_sigma, refined_annuli_sigma, best_coarse_candidate, best_coarse_score = auto_tune_aperture_sigma_grid( + coarse_apertures_sigma, + coarse_annuli_sigma, + coarse_aper_data, + aperture_estimation_comp_count, + tuning_airmass, + require_comp_star=require_comp_star, + skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, + psf_quality_masks=tuning_psf_quality_masks, + ) + refined_tuning_data = populate_aperture_tuning_data_from_cutouts( + tuning_cutouts, + refined_apertures_sigma if use_adaptive_apertures else refined_apertures_sigma * sigma, + refined_annuli_sigma if use_adaptive_apertures else refined_annuli_sigma * sigma, + aperture_estimation_comp_indices, + use_adaptive_apertures, + sigma, + fast_aperture_mask=fast_aperture_mask, + ) + for tuning_frame_index in range(len(tuning_cutouts)): + apply_overexposure_masks_to_aperture_frame( + refined_tuning_data, + tuning_frame_index, + False, + tuning_overexposed_masks, + ) + tuning_selection = select_comparison_calibrated_photometry( + tuning_psf_data, + refined_tuning_data, + refined_apertures_sigma * sigma, + refined_annuli_sigma * sigma, + tuning_airmass, + aperture_estimation_stars, + sigma, + skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, + use_psf_photometry=False, + use_aperture_photometry=True, + comp_overexposed_masks=tuning_overexposed_masks, + ) + if tuning_selection is not None: + selected_aperture_index = int(tuning_selection['a']) + selected_annulus_index = int(tuning_selection['an']) + selected_aperture_sigma = float(refined_apertures_sigma[selected_aperture_index]) + selected_annulus_sigma = float(refined_annuli_sigma[selected_annulus_index]) + if use_adaptive_apertures: + aperture_values = np.asarray([selected_aperture_sigma], dtype=float) + annulus_values = np.asarray([selected_annulus_sigma], dtype=float) + else: + aperture_values = np.asarray([selected_aperture_sigma * sigma], dtype=float) + annulus_values = np.asarray([selected_annulus_sigma * sigma], dtype=float) + apers = np.asarray([selected_aperture_sigma * sigma], dtype=float) + annuli = np.asarray([selected_annulus_sigma * sigma], dtype=float) + aper_data = initialize_aperture_data_store( + len(inputfiles), + 1, + 1, + comp_star_count, + ) + aperture_grid_tuned = True + aperture_preselected_from_sample = True + aperture_tuning_sample_score = float(tuning_selection['field_score']) + best_aper_fwhm = selected_aperture_sigma / GAUSSIAN_SIGMA_TO_FWHM + log_info( + "Distributed aperture tuning selected " + f"aper={selected_aperture_sigma:.2f} sigma/{best_aper_fwhm:.2f} FWHM, " + f"annulus={selected_annulus_sigma:.2f} sigma, " + f"sample_field_score={aperture_tuning_sample_score * 100.0:.4f}%. " + "The selected 1x1 aperture will now be measured for every tracked star on every frame." + ) + del tuning_cutouts + log_info( + "Distributed aperture tuning completed in " + f"{perf_counter() - aperture_tuning_start:.2f}s." + ) + reset_photometry_timing_stats() + use_multiprocess_transform_precompute = False + fallback_transforms = pointing_alignment_transforms + use_memmap_initial_photometry = can_memmap_aperture_tuning_cutouts( + generalDark=generalDark, + generalBias=generalBias, + generalFlat=generalFlat, + demosaic_fmt=demosaic_fmt, + bad_pixel_reference=bad_pixel_reference, + ) + if use_memmap_initial_photometry: + log_info( + "Initial photometry image access: FITS memmap enabled; only tracked-star pixel " + "neighborhoods will be paged in." + ) + initial_photometry_start = perf_counter() + firstImage = None for i, fileName in enumerate(inputfiles): plateStatus.setCurrentFilename(fileName) - hdul = fits.open(name=fileName, memmap=False, cache=False, lazy_load_hdus=False, + frame_uses_memmap = use_memmap_initial_photometry + hdul = fits.open(name=fileName, memmap=frame_uses_memmap, cache=False, + lazy_load_hdus=frame_uses_memmap, ignore_missing_end=True) + # Final reductions should always use the full centroid fit so the + # centroid series does not inherit the fast moment-estimator cadence. + frame_fast_centroid = False + target_fast_centroid = False extension = 0 image_header = hdul[extension].header @@ -2163,115 +34150,825 @@ def main(): extension += 1 image_header = hdul[extension].header + if frame_uses_memmap and not fits_header_supports_memmap(image_header): + hdul.close() + frame_uses_memmap = False + hdul = fits.open( + name=fileName, + memmap=False, + cache=False, + lazy_load_hdus=False, + ignore_missing_end=True, + ) + extension = 0 + image_header = hdul[extension].header + while image_header["NAXIS"] == 0: + extension += 1 + image_header = hdul[extension].header + airMassList.append(air_mass(image_header, pDict['ra'], pDict['dec'], exotic_infoDict['lat'], exotic_infoDict['long'], exotic_infoDict['elev'], jd_times[i])) exptimes.append(get_exp_time(image_header)) + frame_noise_config = noise_budget_config_from_info(exotic_infoDict, image_header) + frame_noise_configs.append(frame_noise_config) + if i == 0: + log_info( + "Photometry noise budget terms: " + f"{format_noise_budget_config_summary(frame_noise_config)}." + ) + frame_airmass = airMassList[-1] + frame_exposure_s = exptimes[-1] # IMAGES imageData = hdul[extension].data # CALS - imageData = apply_cals(imageData, generalDark, generalBias, generalFlat, i) - # Demosaic, if needed - imageData = demosaic_img(imageData, demosaic_fmt, demosaic_out, demosaic_mult, i) + if not frame_uses_memmap: + imageData = apply_cals(imageData, generalDark, generalBias, generalFlat, i) + # Demosaic, if needed + imageData = demosaic_img(imageData, demosaic_fmt, demosaic_out, demosaic_mult, i) + imageData = repair_bad_pixels_in_frame(imageData, bad_pixel_reference) - if i == 0: + if i == 0 and multiprocess_alignment_results is None: firstImage = np.copy(imageData) - sys.stdout.write(f"Finding transformation {i + 1} of {len(inputfiles)} : {fileName}\n") - log.debug(f"Finding transformation {i + 1} of {len(inputfiles)} : {fileName}\n") - sys.stdout.flush() - - try: - wcs_hdr = search_wcs(fileName) - if not wcs_hdr.is_celestial: - raise Exception - - if i == 0: - tx, ty = exotic_UIprevTPX, exotic_UIprevTPY - else: - pix_coords = wcs_hdr.world_to_pixel_values(ra_dec_tar[0], ra_dec_tar[1]) - tx, ty = pix_coords[0].take(0), pix_coords[1].take(0) - - psf_data['target'][i] = fit_centroid(imageData, [tx, ty], 0) - - # TODO: Add check for flux on target/comp stars relative to others in the field - # in case of cloudy data, large changes, etc. - if i != 0 and np.abs((psf_data['target'][i][2] - psf_data['target'][i - 1][2]) - / psf_data['target'][i - 1][2]) > 0.5: - raise Exception - - for j in range(len(exotic_infoDict['comp_stars'])): - ckey = f"comp{j + 1}" - - pix_coords = wcs_hdr.world_to_pixel_values(ra_dec_wcs[j][0], ra_dec_wcs[j][1]) - cx, cy = pix_coords[0].take(0), pix_coords[1].take(0) - psf_data[ckey][i] = fit_centroid(imageData, [cx, cy], j+1) - - if i != 0: - if not (tar_comp_dist[ckey][0] - 1 <= abs(int(psf_data[ckey][i][0]) - int(psf_data['target'][i][0])) <= tar_comp_dist[ckey][0] + 1 and - tar_comp_dist[ckey][1] - 1 <= abs(int(psf_data[ckey][i][1]) - int(psf_data['target'][i][1])) <= tar_comp_dist[ckey][1] + 1) or \ - np.abs((psf_data[ckey][i][2] - psf_data[ckey][i - 1][2]) / psf_data[ckey][i - 1][2]) > 0.5: - raise Exception + if multiprocess_alignment_results is not None: + if not multiprocess_alignment_results_applied: + apply_parallel_alignment_result( + multiprocess_alignment_results[i], + i, + psf_data, + tar_comp_dist, + comp_alignment_keys, + ) + else: + alignment_result = { + 'index': i, + 'file_name': fileName, + 'wcs': None, + 'fallback': None, + } + previous_psf_rows = {} + if i != 0: + previous_psf_rows = {'target': psf_data['target'][i - 1]} + for comp_idx, comp_key in enumerate(comp_alignment_keys): + previous_psf_rows[f"comp{comp_idx + 1}"] = psf_data[comp_key][i - 1] + + has_wcs_alignment = False + if not ignore_header_wcs: + try: + wcs_hdr = search_wcs_from_header(image_header) + has_wcs_alignment = wcs_hdr.is_celestial + except Exception: + has_wcs_alignment = False + + if has_wcs_alignment and target_and_comp_radec is not None: + try: + pix_x, pix_y = wcs_hdr.world_to_pixel_values( + target_and_comp_radec[:, 0], + target_and_comp_radec[:, 1], + ) + pix_x = np.asarray(pix_x, dtype=float).reshape(-1) + pix_y = np.asarray(pix_y, dtype=float).reshape(-1) + projected_coords = np.column_stack((pix_x, pix_y)) + wcs_candidate = _fit_alignment_candidate_psfs( + imageData, + projected_coords, + target_fast_centroid, + frame_fast_centroid, + ) + wcs_candidate['projected_off_frame'] = any_projected_coord_out_of_frame( + projected_coords, + imageData.shape, + ) + alignment_result['wcs'] = wcs_candidate + except Exception: + alignment_result['wcs'] = None + + log_alignment_progress( + i, + len(inputfiles), + fileName, + use_multiprocess_transform_precompute, + pixel_alignment_enabled=pixel_alignment_enabled, + ) + + wcs_candidate_acceptable = wcs_alignment_candidate_is_acceptable( + alignment_result, + i, + psf_data, + tar_comp_dist, + comp_alignment_keys, + ) + if not pixel_alignment_enabled and not wcs_candidate_acceptable: + log_wcs_authoritative_candidate_diagnostics( + alignment_result, + i, + psf_data, + tar_comp_dist, + comp_alignment_keys, + ) + if pixel_alignment_enabled and not wcs_candidate_acceptable: + cached_tform = fallback_transforms.get(str(fileName)) if fallback_transforms else None + if cached_tform is not None: + tform = cached_tform + elif i == 0: + tform = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) else: - tar_comp_dist[ckey][0] = abs(int(psf_data[ckey][0][0]) - int(psf_data['target'][0][0])) - tar_comp_dist[ckey][1] = abs(int(psf_data[ckey][0][1]) - int(psf_data['target'][0][1])) - except Exception: - if i == 0: - tform = SimilarityTransform(scale=1, rotation=0, translation=[0, 0]) + tform = downsampled_fallback_transformation( + imageData, + fileName, + reference_image=firstImage, + ) + + transformed_coords = np.asarray(tform(target_and_comp_pixels), dtype=float) + alignment_result['fallback'] = _fit_alignment_candidate_psfs( + imageData, + transformed_coords, + target_fast_centroid, + frame_fast_centroid, + previous_psf_rows=previous_psf_rows, + ) + apply_parallel_alignment_result( + alignment_result, + i, + psf_data, + tar_comp_dist, + comp_alignment_keys, + ) + + if reject_overexposed_stars: + target_row = np.asarray(psf_data['target'][i], dtype=float) + target_radius = overexposure_aperture_radius_from_psf_row( + target_row, + fallback_sigma=sigma, + ) + if aperture_contains_overexposed_pixel( + imageData, + target_row[0] if target_row.size > 0 else np.nan, + target_row[1] if target_row.size > 1 else np.nan, + target_radius, + overexposure_threshold, + fast_mode=fast_aperture_mask, + ): + target_overexposed_frame_mask[i] = True + plateStatus.overexposedWarning( + 0, + target_row[0] if target_row.size > 0 else np.nan, + target_row[1] if target_row.size > 1 else np.nan, + overexposure_threshold, + ) + psf_flux_data['target'][i, :] = np.nan + + for comp_idx, comp_key in enumerate(comp_alignment_keys): + comp_row = np.asarray(psf_data[comp_key][i], dtype=float) + comp_radius = overexposure_aperture_radius_from_psf_row( + comp_row, + fallback_sigma=sigma, + ) + if aperture_contains_overexposed_pixel( + imageData, + comp_row[0] if comp_row.size > 0 else np.nan, + comp_row[1] if comp_row.size > 1 else np.nan, + comp_radius, + overexposure_threshold, + fast_mode=fast_aperture_mask, + ): + comp_overexposed_masks[comp_key][i] = True + plateStatus.overexposedWarning( + comp_idx + 1, + comp_row[0] if comp_row.size > 0 else np.nan, + comp_row[1] if comp_row.size > 1 else np.nan, + overexposure_threshold, + starLabel=tracked_vsx_labels.get(comp_key), + ) + + if use_psf_photometry: + psf_flux_row_fitter = ( + fit_legacy_psf_photometry_flux_row + if use_legacy_psf_flux_mode + else fit_psf_photometry_flux_row + ) + if target_overexposed_frame_mask[i]: + psf_flux_data['target'][i, :] = np.nan else: - tform = transformation(np.array([imageData, firstImage]), fileName) - - tx, ty = tform([exotic_UIprevTPX, exotic_UIprevTPY])[0] - psf_data['target'][i] = fit_centroid(imageData, [tx, ty], 0) - - for j, coord in enumerate(exotic_infoDict['comp_stars']): - ckey = f"comp{j + 1}" - - cx, cy = tform(coord)[0] - psf_data[ckey][i] = fit_centroid(imageData, [cx, cy], j+1) - - if i == 0: - tar_comp_dist[ckey][0] = abs(int(psf_data[ckey][0][0]) - int(psf_data['target'][0][0])) - tar_comp_dist[ckey][1] = abs(int(psf_data[ckey][0][1]) - int(psf_data['target'][0][1])) + target_psf_flux_seed_row = psf_data['target'][i] + if 'target' in psf_flux_seed_tracks: + target_psf_flux_seed_row = psf_flux_seed_tracks['target'][i] + psf_flux_data['target'][i] = psf_flux_row_fitter( + imageData, + target_psf_flux_seed_row, + 0, + ) + store_psf_noise_budget( + psf_noise_data, + 'target', + i, + compute_psf_noise_budget_for_row( + imageData, + psf_flux_data['target'][i], + 0, + noise_config=frame_noise_config, + exposure_s=frame_exposure_s, + airmass=frame_airmass, + fallback_sigma=sigma, + fast_mode=fast_aperture_mask, + ), + ) + for comp_idx, comp_key in enumerate(comp_alignment_keys): + if comp_overexposed_masks.get(comp_key, np.zeros(len(inputfiles), dtype=bool))[i]: + psf_flux_data[comp_key][i, :] = np.nan + continue + comp_psf_flux_seed_row = psf_data[comp_key][i] + if comp_key in psf_flux_seed_tracks: + comp_psf_flux_seed_row = psf_flux_seed_tracks[comp_key][i] + psf_flux_data[comp_key][i] = psf_flux_row_fitter( + imageData, + comp_psf_flux_seed_row, + comp_idx + 1, + ) + store_psf_noise_budget( + psf_noise_data, + comp_key, + i, + compute_psf_noise_budget_for_row( + imageData, + psf_flux_data[comp_key][i], + comp_idx + 1, + noise_config=frame_noise_config, + exposure_s=frame_exposure_s, + airmass=frame_airmass, + fallback_sigma=sigma, + fast_mode=fast_aperture_mask, + ), + ) # aperture photometry - if i == 0: - sigma = float((psf_data['target'][0][3] + psf_data['target'][0][4]) * 0.5) - apers *= sigma - annuli *= sigma - - for a, aper in enumerate(apers): - for an, annulus in enumerate(annuli): - if not np.isnan(psf_data['target'][i, 0]): - aper_data["target"][i][a][an], aper_data["target_bg"][i][a][an] = aperPhot(imageData, 0, - psf_data['target'][i, 0], - psf_data['target'][i, 1], - aper, annulus) + if use_aperture_photometry and i == 0 and not aperture_preselected_from_sample: + sigma = aperture_frame_sigma_from_psf_data( + psf_data, + 0, + comparison_indices=aperture_frame_sigma_comp_indices, + ) + if use_aperture_corrections_and_full_image_fwhm: + image_fwhm = estimate_image_fwhm_from_isolated_stars( + imageData, + fwhm_hint=psf_fwhm_from_sigma(sigma), + fallback_sigma=sigma, + ) + if np.isfinite(image_fwhm) and image_fwhm > 0: + sigma = image_fwhm / GAUSSIAN_SIGMA_TO_FWHM + if not np.isfinite(sigma) or sigma <= 0: + log_info("Warning: Initial PSF sigma is invalid; using sigma=1.0 for automatic aperture tuning.", warn=True) + sigma = 1.0 + if use_adaptive_apertures: + coarse_aperture_values = coarse_apertures_sigma + coarse_annulus_values = coarse_annuli_sigma + else: + coarse_aperture_values = coarse_apertures_sigma * sigma + coarse_annulus_values = coarse_annuli_sigma * sigma + + if use_aperture_photometry and aperture_preselected_from_sample: + populate_aperture_data_for_frame( + imageData, + i, + psf_data, + comp_star_count, + aper_data, + aperture_values, + annulus_values, + fast_aperture_mask, + adaptive_apertures=use_adaptive_apertures, + fallback_sigma=sigma, + use_aperture_corrections_and_full_image_fwhm=False, + noise_config=frame_noise_config, + exposure_s=frame_exposure_s, + airmass=frame_airmass, + comp_indices=range(comp_star_count), + include_target=True, + frame_sigma_comp_indices=aperture_frame_sigma_comp_indices, + ) + apply_overexposure_masks_to_aperture_frame( + aper_data, + i, + target_overexposed_frame_mask[i], + comp_overexposed_masks, + ) + elif use_aperture_photometry and i < coarse_tune_frames: + coarse_frame_cache[i] = np.array(imageData, copy=True) + populate_aperture_data_for_frame( + imageData, + i, + psf_data, + comp_star_count, + coarse_aper_data, + coarse_aperture_values, + coarse_annulus_values, + fast_aperture_mask, + adaptive_apertures=use_adaptive_apertures, + fallback_sigma=sigma, + use_aperture_corrections_and_full_image_fwhm=use_aperture_corrections_and_full_image_fwhm, + noise_config=frame_noise_config, + exposure_s=frame_exposure_s, + airmass=frame_airmass, + comp_indices=aperture_estimation_comp_indices, + include_target=aperture_estimation_includes_target, + frame_sigma_comp_indices=aperture_frame_sigma_comp_indices, + ) + apply_overexposure_masks_to_aperture_frame( + coarse_aper_data, + i, + target_overexposed_frame_mask[i], + comp_overexposed_masks, + ) + + if i == coarse_tune_frames - 1: + subset_airmass = np.asarray(airMassList[:coarse_tune_frames], dtype=float) + coarse_psf_quality_masks = { + f"comp{comp_idx + 1}": psf_quality_mask_for_key( + {key: value[:coarse_tune_frames] for key, value in psf_data.items()}, + f"comp{comp_idx + 1}", + coarse_tune_frames, + ) + for comp_idx in range(aperture_estimation_comp_count) + } + refined_apertures_sigma, refined_annuli_sigma, best_coarse_candidate, best_coarse_score = auto_tune_aperture_sigma_grid( + coarse_apertures_sigma, + coarse_annuli_sigma, + coarse_aper_data, + aperture_estimation_comp_count, + subset_airmass, + require_comp_star=require_comp_star, + skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, + psf_quality_masks=coarse_psf_quality_masks, + ) + if use_adaptive_apertures: + aperture_values = refined_apertures_sigma + annulus_values = refined_annuli_sigma + else: + aperture_values = refined_apertures_sigma * sigma + annulus_values = refined_annuli_sigma * sigma + apers = refined_apertures_sigma * sigma + annuli = refined_annuli_sigma * sigma + aper_data = initialize_aperture_data_store(len(inputfiles), len(apers), len(annuli), comp_star_count) + aperture_grid_tuned = True + + best_comp_label = "none" + if best_coarse_candidate['comp_index'] is not None: + best_comp_label = str(best_coarse_candidate['comp_index'] + 1) + score_text = "n/a" if not np.isfinite(best_coarse_score) else f"{best_coarse_score:.5f}" + best_aper_sigma = best_coarse_candidate['aper_sigma'] + best_aper_fwhm = best_aper_sigma / GAUSSIAN_SIGMA_TO_FWHM + log_info( + "Auto-tuned aperture grid: " + f"coarse_best=(aper={best_aper_sigma:.2f} sigma/{best_aper_fwhm:.2f} FWHM, " + f"annulus={best_coarse_candidate['annulus_sigma']:.2f} sigma, comp={best_comp_label}, score={score_text}), " + f"refined_grid={len(refined_apertures_sigma)}x{len(refined_annuli_sigma)}." + ) + + log_info(f"Backfilling refined aperture photometry for the first {coarse_tune_frames} frame(s).") + for backfill_idx in range(coarse_tune_frames): + if ( + aperture_estimation_includes_target + and target_overexposed_frame_mask[backfill_idx] + ): + apply_overexposure_masks_to_aperture_frame( + aper_data, + backfill_idx, + True, + comp_overexposed_masks, + ) + coarse_frame_cache[backfill_idx] = None + continue + backfill_image = coarse_frame_cache[backfill_idx] + loaded_from_disk = False + if backfill_image is None: + backfill_image = load_calibrated_reduction_image( + inputfiles[backfill_idx], + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + ) + loaded_from_disk = True + try: + populate_aperture_data_for_frame( + backfill_image, + backfill_idx, + psf_data, + comp_star_count, + aper_data, + aperture_values, + annulus_values, + fast_aperture_mask, + adaptive_apertures=use_adaptive_apertures, + fallback_sigma=sigma, + use_aperture_corrections_and_full_image_fwhm=( + use_aperture_corrections_and_full_image_fwhm + ), + noise_config=frame_noise_configs[backfill_idx], + exposure_s=exptimes[backfill_idx], + airmass=airMassList[backfill_idx], + comp_indices=aperture_estimation_comp_indices, + include_target=aperture_estimation_includes_target, + frame_sigma_comp_indices=aperture_frame_sigma_comp_indices, + ) + apply_overexposure_masks_to_aperture_frame( + aper_data, + backfill_idx, + target_overexposed_frame_mask[backfill_idx], + comp_overexposed_masks, + ) + finally: + if loaded_from_disk: + del backfill_image + coarse_frame_cache[backfill_idx] = None + elif use_aperture_photometry: + if not aperture_grid_tuned: + # Defensive fallback for unexpected control flow. + aperture_values = coarse_aperture_values + annulus_values = coarse_annulus_values + if use_adaptive_apertures: + apers = coarse_apertures_sigma * sigma + annuli = coarse_annuli_sigma * sigma else: - aper_data["target"][i][a][an] = np.nan - aper_data["target_bg"][i][a][an] = np.nan - # loop through comp stars - for j in range(len(exotic_infoDict['comp_stars'])): - ckey = f"comp{j + 1}" - if not np.isnan(psf_data[ckey][i][0]): - aper_data[ckey][i][a][an], \ - aper_data[f"{ckey}_bg"][i][a][an] = aperPhot(imageData, j + 1, psf_data[ckey][i, 0], - psf_data[ckey][i, 1], aper, annulus) - else: - aper_data[ckey][i][a][an] = np.nan - aper_data[f"{ckey}_bg"][i][a][an] = np.nan + apers = coarse_aperture_values + annuli = coarse_annulus_values + aper_data = initialize_aperture_data_store(len(inputfiles), len(apers), len(annuli), comp_star_count) + aperture_grid_tuned = True + + populate_aperture_data_for_frame( + imageData, + i, + psf_data, + comp_star_count, + aper_data, + aperture_values, + annulus_values, + fast_aperture_mask, + adaptive_apertures=use_adaptive_apertures, + fallback_sigma=sigma, + use_aperture_corrections_and_full_image_fwhm=use_aperture_corrections_and_full_image_fwhm, + noise_config=frame_noise_config, + exposure_s=frame_exposure_s, + airmass=frame_airmass, + comp_indices=aperture_estimation_comp_indices, + include_target=aperture_estimation_includes_target, + frame_sigma_comp_indices=aperture_frame_sigma_comp_indices, + ) + apply_overexposure_masks_to_aperture_frame( + aper_data, + i, + target_overexposed_frame_mask[i], + comp_overexposed_masks, + ) # close file + delete from memory hdul.close() del hdul del imageData + completed_photometry_frames = i + 1 + if completed_photometry_frames == len(inputfiles) or completed_photometry_frames % 50 == 0: + log_info( + "Initial photometry progress: " + f"{completed_photometry_frames}/{len(inputfiles)} frame(s)." + ) + + log_info( + "Initial selected-aperture/PSF photometry completed in " + f"{perf_counter() - initial_photometry_start:.2f}s." + ) + plateStatus.logAggregatedWarningSummary() + log_transform_timing_stats('Transformation timing summary (full reduction)') + log_photometry_timing_stats('Photometry timing summary (full reduction)') + log_reduction_timing_overview('Reduction timing overview (full reduction)') + reduction_stage_timer.checkpoint("WCS alignment, centroiding, and initial frame photometry") + + frozen_aperture_data = None + frozen_aperture_values = None + frozen_annulus_values = None + frozen_apers = None + frozen_annuli = None + frozen_backfill_comp_indices = tuple(range(aperture_estimation_comp_count, comp_star_count)) + if ( + use_aperture_photometry + and aper_data is not None + and aperture_values is not None + and annulus_values is not None + and (frozen_backfill_comp_indices or stellar_variability_only) + ): + full_airmass = np.asarray(airMassList, dtype=float) + aperture_reference_sigmas = np.asarray([ + aperture_frame_sigma_from_psf_data( + psf_data, + frame_index, + comparison_indices=aperture_frame_sigma_comp_indices, + ) + for frame_index in range(len(inputfiles)) + ], dtype=float) + aperture_reference_sigmas = aperture_reference_sigmas[ + np.isfinite(aperture_reference_sigmas) & (aperture_reference_sigmas > 0) + ] + aperture_reference_sigma = ( + float(np.median(aperture_reference_sigmas)) + if aperture_reference_sigmas.size + else finite_positive_or_nan(sigma) + ) + if not np.isfinite(aperture_reference_sigma) or aperture_reference_sigma <= 0: + aperture_reference_sigma = 1.0 + if use_adaptive_apertures: + aperture_grid_apers = np.asarray(aperture_values, dtype=float) * aperture_reference_sigma + aperture_grid_annuli = np.asarray(annulus_values, dtype=float) * aperture_reference_sigma + else: + aperture_grid_apers = np.asarray(aperture_values, dtype=float) + aperture_grid_annuli = np.asarray(annulus_values, dtype=float) + + aperture_estimation_calibration = select_comparison_calibrated_photometry( + psf_data, + aper_data, + aperture_grid_apers, + aperture_grid_annuli, + full_airmass, + aperture_estimation_stars, + aperture_reference_sigma, + skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, + use_psf_photometry=False, + use_aperture_photometry=True, + comp_overexposed_masks=comp_overexposed_masks, + ) + if aperture_estimation_calibration is None: + log_info( + "Warning: science comparison stars did not yield a usable aperture-grid " + "selection, so additional tracked stars cannot be aperture-photometered " + "with a frozen science aperture. PSF photometry remains available when enabled.", + warn=True, + ) + else: + selected_aperture_index = int(aperture_estimation_calibration['a']) + selected_annulus_index = int(aperture_estimation_calibration['an']) + frozen_aperture_values = np.asarray([ + np.asarray(aperture_values, dtype=float)[selected_aperture_index] + ]) + frozen_annulus_values = np.asarray([ + np.asarray(annulus_values, dtype=float)[selected_annulus_index] + ]) + frozen_apers = np.asarray([aperture_estimation_calibration['aper']], dtype=float) + frozen_annuli = np.asarray([aperture_estimation_calibration['annulus']], dtype=float) + frozen_aperture_data = collapse_aperture_data_to_selected_grid_cell( + aper_data, + selected_aperture_index, + selected_annulus_index, + ) + estimator_description = ( + "the first five bright, reference-frame non-saturated, VSX-vetted " + "non-variable comparison stars" + if ( + stellar_variability_only + and use_ensemble_photometry_for_stellar_variability + and aperture_estimation_comp_count == 5 + ) + else f"{aperture_estimation_comp_count} science comparison star(s)" + ) + if aperture_preselected_from_sample: + score_delta = ( + aperture_estimation_calibration['field_score'] - aperture_tuning_sample_score + if np.isfinite(aperture_tuning_sample_score) + else np.nan + ) + delta_text = ( + f", delta={score_delta * 100.0:+.4f}%" + if np.isfinite(score_delta) + else "" + ) + log_info( + "Full-sequence aperture validation: " + f"aper={frozen_apers[0]:.2f}px, annulus={frozen_annuli[0]:.2f}px, " + f"field_score={aperture_estimation_calibration['field_score'] * 100.0:.4f}%" + f"{delta_text}. Target and all {comp_star_count} tracked star(s) were already " + "measured in the initial pass; frozen-aperture reread skipped." + ) + else: + log_info( + "Frozen aperture selected from the science reduction grid using " + f"{estimator_description}: aper={frozen_apers[0]:.2f}px, " + f"annulus={frozen_annuli[0]:.2f}px. " + + ( + "Measuring the variable science target once and " + if stellar_variability_only + else "Measuring " + ) + + f"{len(frozen_backfill_comp_indices)} additional tracked star(s) once " + "with this setup." + ) + + reset_photometry_timing_stats() + frozen_backfill_total = len(inputfiles) + for frozen_frame_index, frozen_file_name in enumerate(inputfiles): + active_comp_indices = [ + comp_idx + for comp_idx in frozen_backfill_comp_indices + if not comp_overexposed_masks.get( + f"comp{comp_idx + 1}", + np.zeros(frozen_backfill_total, dtype=bool), + )[frozen_frame_index] + ] + measure_frozen_target = bool( + stellar_variability_only + and not target_overexposed_frame_mask[frozen_frame_index] + ) + if active_comp_indices or measure_frozen_target: + frozen_image = load_calibrated_reduction_image( + frozen_file_name, + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + ) + try: + populate_aperture_data_for_frame( + frozen_image, + frozen_frame_index, + psf_data, + comp_star_count, + frozen_aperture_data, + frozen_aperture_values, + frozen_annulus_values, + fast_aperture_mask, + adaptive_apertures=use_adaptive_apertures, + fallback_sigma=sigma, + use_aperture_corrections_and_full_image_fwhm=( + use_aperture_corrections_and_full_image_fwhm + ), + noise_config=frame_noise_configs[frozen_frame_index], + exposure_s=exptimes[frozen_frame_index], + airmass=airMassList[frozen_frame_index], + comp_indices=active_comp_indices, + include_target=measure_frozen_target, + frame_sigma_comp_indices=aperture_frame_sigma_comp_indices, + ) + finally: + del frozen_image + apply_overexposure_masks_to_aperture_frame( + frozen_aperture_data, + frozen_frame_index, + bool( + stellar_variability_only + and target_overexposed_frame_mask[frozen_frame_index] + ), + comp_overexposed_masks, + ) + completed_frozen_frames = frozen_frame_index + 1 + if ( + completed_frozen_frames == frozen_backfill_total + or completed_frozen_frames % 50 == 0 + ): + log_info( + "Frozen-aperture photometry progress: " + f"{completed_frozen_frames}/{frozen_backfill_total}" + ) + log_photometry_timing_stats( + 'Photometry timing summary (frozen-aperture additional stars)' + ) + + # Fortuitous VSX targets are independent science targets. Process them against + # the full image sequence before the exoplanet target validity/overexposure mask + # is applied below. Each VSX target series applies its own compN overexposure mask. + if photometer_fortuitous_variables and fortuitous_variables: + full_airmass = np.asarray(airMassList, dtype=float) + full_exposure_times_seconds = np.asarray(exptimes, dtype=float) + variable_sigma_rows = [ + np.asarray(psf_data.get(variable.get('tracking_key'), []), dtype=float) + for variable in fortuitous_variables + if variable.get('tracking_key') in psf_data + ] + variable_sigma_rows = [rows for rows in variable_sigma_rows if rows.ndim == 2 and rows.size] + if variable_sigma_rows: + fortuitous_sigma_display = representative_psf_sigma( + np.concatenate(variable_sigma_rows, axis=0), + fallback_sigma=sigma, + ) + else: + fortuitous_sigma_display = sigma + if not np.isfinite(fortuitous_sigma_display) or fortuitous_sigma_display <= 0: + fortuitous_sigma_display = 1.0 + + fortuitous_aperture_data = frozen_aperture_data + fortuitous_apers = frozen_apers + fortuitous_annuli = frozen_annuli + if fortuitous_aperture_data is None: + fortuitous_aperture_data = aper_data + fortuitous_apers = apers + fortuitous_annuli = annuli + if ( + frozen_aperture_data is None + and aperture_values is not None + and annulus_values is not None + ): + if use_adaptive_apertures: + fortuitous_apers = ( + np.asarray(aperture_values, dtype=float) * fortuitous_sigma_display + ) + fortuitous_annuli = ( + np.asarray(annulus_values, dtype=float) * fortuitous_sigma_display + ) + else: + fortuitous_apers = np.asarray(aperture_values, dtype=float) + fortuitous_annuli = np.asarray(annulus_values, dtype=float) + + fortuitous_psf_flux_source = ( + psf_flux_data if use_psf_photometry else psf_data + ) + fortuitous_comparison_calibration = select_comparison_calibrated_photometry( + psf_data, + fortuitous_aperture_data, + fortuitous_apers, + fortuitous_annuli, + full_airmass, + fortuitous_ensemble_stars, + fortuitous_sigma_display, + skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, + use_psf_photometry=use_psf_photometry, + use_aperture_photometry=use_aperture_photometry, + psf_flux_data=fortuitous_psf_flux_source, + comp_overexposed_masks=comp_overexposed_masks, + ) + exotic_infoDict['exposure'] = exp_time_med(exptimes) + process_fortuitous_variables( + fortuitous_variables, + fortuitous_comparison_calibration, + fortuitous_calibration_stars, + times, + jd_times, + full_airmass, + psf_data, + fortuitous_aperture_data, + exotic_infoDict, + psf_flux_data=fortuitous_psf_flux_source, + psf_noise_data=psf_noise_data if use_psf_photometry else None, + comp_overexposed_masks=comp_overexposed_masks, + exposure_times_seconds=full_exposure_times_seconds, + observed_filter=exotic_infoDict.get( + 'observed_filter', + exotic_infoDict.get('filter'), + ), + use_single_comparison=use_single_comparison_for_fortuitous_variables, + maximum_number_of_ensemble_comparisons_for_stellar_variability= + maximum_number_of_ensemble_comparisons_for_stellar_variability, + ) + + reduction_stage_timer.checkpoint("Aperture finalization and fortuitous-variable photometry") + + if stellar_variability_only and frozen_aperture_data is not None: + aper_data = frozen_aperture_data + apers = frozen_apers + annuli = frozen_annuli + aperture_values = frozen_aperture_values + annulus_values = frozen_annulus_values + # filter bad images - badmask = np.isnan(psf_data["target"][:, 0]) | (psf_data["target"][:, 0] == 0) | (aper_data["target"][:, 0, 0] == 0) | np.isnan( - aper_data["target"][:, 0, 0]) + badmask = np.isnan(psf_data["target"][:, 0]) | (psf_data["target"][:, 0] == 0) + if aper_data is not None: + badmask = badmask | (aper_data["target"][:, 0, 0] == 0) | np.isnan(aper_data["target"][:, 0, 0]) + if reject_overexposed_stars and target_overexposed_frame_mask.shape == badmask.shape: + target_overexposure_diagnostic = build_time_rejection_diagnostic( + "Target overexposure filter", + times, + ~target_overexposed_frame_mask, + note=( + "Dropped frames before photometry selection because one or more target aperture pixels " + f"exceeded {overexposure_threshold:.1f} " + f"({overexposure_threshold_fraction:.3f} of saturation_value={saturation_value:.1f})." + ), + ) + if ( + target_overexposure_diagnostic is not None + and target_overexposure_diagnostic['dropped_point_count'] > 0 + ): + log_lightcurve_filter_diagnostics( + [target_overexposure_diagnostic], + header="Target overexposure frame rejections before photometry selection", + ) + badmask = badmask | target_overexposed_frame_mask goodmask = ~badmask + global_frame_filter_diagnostic = build_time_rejection_diagnostic( + "Target centroid/aperture validity filter", + times, + goodmask, + note="Dropped frames before photometry selection because the target centroid or target aperture photometry was invalid.", + ) + if global_frame_filter_diagnostic is not None and global_frame_filter_diagnostic['dropped_point_count'] > 0: + log_lightcurve_filter_diagnostics( + [global_frame_filter_diagnostic], + header="Global reduction frame rejections before photometry selection", + ) if np.sum(goodmask) == 0: log_info("No images to fit...check reference image for alignment (first image of sequence)") @@ -2279,25 +34976,131 @@ def main(): times = times[goodmask] jd_times = jd_times[goodmask] airmass = np.array(airMassList)[goodmask] + exposure_times_seconds = np.asarray(exptimes, dtype=float)[goodmask] + exptimes = exposure_times_seconds.tolist() psf_data["target"] = psf_data["target"][goodmask] - aper_data["target"] = aper_data["target"][goodmask] - aper_data["target_bg"] = aper_data["target_bg"][goodmask] + psf_flux_data["target"] = psf_flux_data["target"][goodmask] + for key in list(psf_noise_data.keys()): + psf_noise_data[key] = psf_noise_data[key][goodmask] + if aper_data is not None: + for key in list(aper_data.keys()): + aper_data[key] = aper_data[key][goodmask] + target_overexposed_frame_mask = target_overexposed_frame_mask[goodmask] + comp_overexposed_masks = { + key: np.asarray(mask, dtype=bool)[goodmask] + for key, mask in comp_overexposed_masks.items() + } for j in range(len(exotic_infoDict['comp_stars'])): ckey = f"comp{j + 1}" psf_data[ckey] = psf_data[ckey][goodmask] - aper_data[ckey] = aper_data[ckey][goodmask] - aper_data[f"{ckey}_bg"] = aper_data[f"{ckey}_bg"][goodmask] + psf_flux_data[ckey] = psf_flux_data[ckey][goodmask] + + psf_quality_diagnostics = [] + if use_psf_photometry: + target_quality_components = target_psf_shape_quality_components(psf_flux_data['target']) + else: + target_quality_components = target_psf_shape_quality_components(psf_data['target']) + target_quality_keep_mask = target_quality_components['keep_mask'] + if target_quality_keep_mask.shape == times.shape and np.any(~target_quality_keep_mask): + reason_parts = [] + invalid_count = int(np.count_nonzero(target_quality_components['invalid_mask'])) + seeing_count = int(np.count_nonzero(target_quality_components['seeing_outlier_mask'])) + axis_ratio_count = int(np.count_nonzero(target_quality_components['axis_ratio_outlier_mask'])) + if invalid_count: + reason_parts.append(f"invalid PSF={invalid_count}") + if seeing_count: + reason_parts.append(f"broad PSF outlier={seeing_count}") + if axis_ratio_count: + reason_parts.append(f"elongated PSF={axis_ratio_count}") + reason_text = "; ".join(reason_parts) + psf_quality_diagnostics.append(build_time_rejection_diagnostic( + "Target PSF shape quality filter", + times, + target_quality_keep_mask, + note=( + "Dropped target frame-level PSF photometry before target/comparison fitting " + f"based on robust PSF shape diagnostics ({reason_text})." + ), + )) + for key, label in [ + (f"comp{j + 1}", f"Comp {j + 1}") for j in range(len(exotic_infoDict['comp_stars'])) + ]: + quality_rows = psf_quality_rows_for_key( + psf_data, + key, + psf_flux_data=psf_flux_data if use_psf_photometry else None, + ) + quality_components = psf_frame_quality_components(quality_rows) + quality_keep_mask = quality_components['keep_mask'] + if quality_keep_mask.shape == times.shape and np.any(~quality_keep_mask): + reason_parts = [] + invalid_count = int(np.count_nonzero(quality_components['invalid_mask'])) + seeing_count = int(np.count_nonzero(quality_components['seeing_outlier_mask'])) + amplitude_count = int(np.count_nonzero(quality_components['amplitude_outlier_mask'])) + if invalid_count: + reason_parts.append(f"invalid PSF={invalid_count}") + if seeing_count: + reason_parts.append(f"seeing outlier={seeing_count}") + if amplitude_count: + reason_parts.append(f"low amplitude outlier={amplitude_count}") + reason_text = "; ".join(reason_parts) + psf_quality_diagnostics.append(build_time_rejection_diagnostic( + f"{label} PSF seeing/amplitude quality filter", + times, + quality_keep_mask, + note=( + "Dropped this star's frame-level photometry before comparison-star " + "intercomparison scoring " + f"based on robust PSF diagnostics ({reason_text})." + ), + )) + if psf_quality_diagnostics: + log_lightcurve_filter_diagnostics( + psf_quality_diagnostics, + header="PSF frame rejections before comparison-star calibration", + ) + + sigma_display = representative_psf_sigma(psf_data['target'], fallback_sigma=sigma) + if not np.isfinite(sigma_display) or sigma_display <= 0: + sigma_display = 1.0 + + if aperture_values is not None and annulus_values is not None: + if use_adaptive_apertures: + apers = np.asarray(aperture_values, dtype=float) * sigma_display + annuli = np.asarray(annulus_values, dtype=float) * sigma_display + else: + apers = np.asarray(aperture_values, dtype=float) + annuli = np.asarray(annulus_values, dtype=float) exotic_infoDict['exposure'] = exp_time_med(exptimes) # save PSF data to disk using savetxt - np.savetxt(Path(exotic_infoDict['save']) / "temp" / "psf_data_target.txt", psf_data["target"], + working_artifacts_dir = Path(exotic_infoDict['save']) / "working_artifacts" + psf_flux_artifacts_dir = working_artifacts_dir / "psf_flux_data" + np.savetxt(working_artifacts_dir / "psf_data_target.txt", psf_data["target"], header="#x_centroid, y_centroid, amplitude, sigma_x, sigma_y, rotation offset", fmt="%.6f") # x-cent, y-cent, amplitude, sigma-x, sigma-y, rotation, offset + if use_psf_photometry: + psf_flux_artifacts_dir.mkdir(parents=True, exist_ok=True) + np.savetxt( + psf_flux_artifacts_dir / "psf_flux_data_target.txt", + psf_flux_data["target"], + header="#x_centroid, y_centroid, amplitude, sigma_x, sigma_y, rotation offset", + fmt="%.6f", + ) + for j in range(len(exotic_infoDict['comp_stars'])): + ckey = f"comp{j + 1}" + np.savetxt( + psf_flux_artifacts_dir / f"psf_flux_data_{ckey}.txt", + psf_flux_data[ckey], + header="#x_centroid, y_centroid, amplitude, sigma_x, sigma_y, rotation offset", + fmt="%.6f", + ) # PSF flux - tFlux = 2 * np.pi * psf_data['target'][:, 2] * psf_data['target'][:, 3] * psf_data['target'][:, 4] + psf_flux_source = psf_flux_data if use_psf_photometry else psf_data + tFlux = psf_flux_series_from_rows(psf_flux_source['target']) ref_flux = {} if vsp_list: @@ -2309,6 +35112,11 @@ def main(): 'flux_unc_tar': None, 'flux_unc_ref': None } + fallback_gain_e_per_adu = ( + frame_noise_configs[0].get('gain_e_per_adu') + if frame_noise_configs + else None + ) centroid_positions = { 'x_targ': None, @@ -2321,190 +35129,999 @@ def main(): 'best_fit_lc': None, 'comp_star_num': None, 'comp_star_coords': None, - 'min_std': 100000, 'min_aperture': None, - 'min_annulus': None + 'min_annulus': None, + 'aperture_index': None, + 'annulus_index': None, + 'adaptive_summary': None, + 'calibration_field_score': np.inf, + 'selection_basis': 'target_fit', + 'selection_metric': 'ktmf', + 'comparison_ktmf_metric': np.nan, + 'comparison_eebls_snr': np.nan, + 'comparison_transit_delta_bic': np.nan, + 'noise_budget_summary': ( + format_noise_budget_config_summary(frame_noise_configs[0]) + if frame_noise_configs else None + ), + 'noise_budget_terms': ( + list(frame_noise_configs[0].get('enabled_terms', ())) + if frame_noise_configs else [] + ), } - # loop over comp stars - for j in range(len(exotic_infoDict['comp_stars'])): - ckey = f"comp{j + 1}" - - cFlux = 2 * np.pi * psf_data[ckey][:, 2] * psf_data[ckey][:, 3] * psf_data[ckey][:, 4] - myfit, tFlux1, cFlux1 = fit_lightcurve(times, tFlux, cFlux, airmass, ld, pDict, jd_times) - - if myfit is not None: - for k in myfit.bounds.keys(): - log.debug(f" {k}: {myfit.parameters[k]:.6f}") - - log.debug("The Residual Standard Deviation is: " - f"{round(100 * myfit.residuals.std() / np.median(myfit.data), 6)}%") - log.debug(f"The Mean Squared Error is: {round(np.sum(myfit.residuals ** 2), 6)}\n") + comparison_calibration = None + stellar_variability_output_selection = None + comparison_calibration = select_comparison_calibrated_photometry( + psf_data, + aper_data, + apers, + annuli, + airmass, + science_comp_stars, + sigma_display, + skip_low_comparison_coverage_rejection=( + skip_low_comp_coverage_rejection or use_exactly_the_comps_provided + ), + use_psf_photometry=use_psf_photometry, + use_aperture_photometry=use_aperture_photometry, + psf_flux_data=psf_flux_source, + comp_overexposed_masks=( + None if use_exactly_the_comps_provided else comp_overexposed_masks + ), + use_exactly_the_comps_provided=use_exactly_the_comps_provided, + ) - res_std = myfit.residuals.std() / np.median(myfit.data) + # Fortuitous-only sources remain in the shared centroid/PSF tracks and frozen-aperture store. + # Restore the science comparison list before normal target fitting and final metadata output. + exotic_infoDict['comp_stars'] = [list(position) for position in science_comp_stars] + + if comparison_calibration is not None: + log_info("\nCalibrating comparison stars before target fitting. Please wait.") + log_info(f"Comparison-star field method: {comparison_calibration['method_label']}") + log_info(f"Comparison-star field score: {comparison_calibration['field_score'] * 100.0:.4f}%") + if comparison_calibration.get('suitability_outlier_rejected_count', 0) > 0: + threshold = comparison_calibration.get('suitability_high_threshold', np.nan) + if np.isfinite(threshold): + log_info( + "Comparison-star field sigma clipping rejected " + f"{comparison_calibration['suitability_outlier_rejected_count']} high-suitability " + f"outlier(s) above {threshold * 100.0:.4f}% before target-fit evaluation." + ) + else: + log_info( + "Comparison-star field sigma clipping rejected " + f"{comparison_calibration['suitability_outlier_rejected_count']} high-suitability " + "outlier(s) before target-fit evaluation." + ) + if comparison_calibration.get('image_outlier_rejected_count', 0) > 0: + required_pairs = comparison_calibration.get('image_outlier_required_valid_pairs', 0) + sigma_threshold = comparison_calibration.get('image_outlier_sigma', COMPARISON_IMAGE_OUTLIER_SIGMA) + log_info( + "Comparison-star field image clipping rejected " + f"{comparison_calibration['image_outlier_rejected_count']} frame(s) after suitability clipping " + f"because every valid pairwise comparison was more than {sigma_threshold:.2f} sigma from " + f"its flat-line median (min valid pair count={required_pairs})." + ) + for summary in comparison_calibration['comp_summaries']: + aggregate_text = "n/a" if not np.isfinite(summary['aggregate_score']) else f"{summary['aggregate_score'] * 100.0:.4f}%" + intercomparison_text = "n/a" if not np.isfinite(summary['ensemble_score']) else f"{summary['ensemble_score'] * 100.0:.4f}%" + pairwise_text = "n/a" if not np.isfinite(summary['pairwise_median_score']) else f"{summary['pairwise_median_score'] * 100.0:.4f}%" + selected_label = " [selected]" if summary['selected'] else "" + position_text = format_comp_star_position(summary['position']) + coverage_text = f"coverage={format_comp_star_coverage_text(summary)}" + if summary['coverage_rejected']: + coverage_text += " [rejected: low coverage]" + if summary.get('suitability_outlier_rejected'): + coverage_text += " [rejected: high suitability outlier]" + intercomparison_frame_text = "" + if summary.get('ensemble_frame_rejected_count', 0) > 0: + intercomparison_frame_text = ( + f", intercomparison_frame_rejects={summary['ensemble_frame_rejected_count']}" + ) + psf_quality_text = "" + if summary.get('psf_quality_rejected_count', 0) > 0: + psf_quality_text = ( + f", psf_quality_rejects={summary['psf_quality_rejected_count']}" + ) + overexposure_text = "" + if summary.get('overexposure_rejected_count', 0) > 0: + overexposure_text = ( + f", overexposure_rejects={summary['overexposure_rejected_count']}" + ) + log_info( + f" {summary['label']}{selected_label} ({position_text}): suitability={aggregate_text}, " + f"intercomparison={intercomparison_text}, pairwise_median={pairwise_text}, " + f"valid_pairs={summary['valid_pair_count']}, {coverage_text}" + f"{psf_quality_text}{overexposure_text}{intercomparison_frame_text}, " + f"reason={summary['selection_reason']}" + ) - if photometry_info['min_std'] > res_std and myfit is not None: - photometry_info.update(best_fit_lc=copy.deepcopy(myfit), - comp_star_num=j + 1, comp_star_coords=exotic_infoDict['comp_stars'][j], - min_std=res_std, min_aperture=0, min_annulus=15 * sigma) + try: + plot_comp_star_pairwise_matrix( + comparison_calibration['pairwise_matrix'], + comparison_calibration['best_comp_index'], + pDict['pName'], + exotic_infoDict['save'], + exotic_infoDict['date'], + comparison_calibration['method_label'], + ) + plot_comp_star_calibration_series( + times, + comparison_calibration['comp_summaries'], + pDict['pName'], + exotic_infoDict['save'], + exotic_infoDict['date'], + comparison_calibration['method_label'], + ) + plot_individual_comp_star_calibration_series( + times, + comparison_calibration['comp_summaries'], + pDict['pName'], + exotic_infoDict['save'], + exotic_infoDict['date'], + comparison_calibration['method_label'], + ) + plot_comp_star_suitability( + comparison_calibration['comp_summaries'], + pDict['pName'], + exotic_infoDict['save'], + exotic_infoDict['date'], + comparison_calibration['method_label'], + ) + save_comp_star_calibration_summary( + exotic_infoDict['save'], + pDict['pName'], + exotic_infoDict['date'], + comparison_calibration['method_label'], + comparison_calibration['field_score'], + comparison_calibration['comp_summaries'], + comparison_calibration['best_comp_index'], + ) + except Exception as e: + log_info(f"Warning: Could not save comparison-star calibration outputs ({e}).", warn=True) + + if stellar_variability_only: + log_info( + "Stellar-variability-only mode: selecting comparison photometry by " + "out-of-transit target/reference scatter without fitting transit models." + ) + comparison_fit_search = select_stellar_variability_only_photometry( + times, + jd_times, + airmass, + pDict, + comparison_calibration, + psf_data, + aper_data, + tFlux, + psf_flux_data=psf_flux_source, + psf_noise_data=psf_noise_data if use_psf_photometry else None, + plot_time_range=full_plot_time_range, + use_adaptive_apertures=use_adaptive_apertures, + adaptive_aperture_values=aperture_values, + adaptive_annulus_values=annulus_values, + fallback_sigma=sigma_display, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=fallback_gain_e_per_adu, + use_ensemble_photometry=use_ensemble_photometry_for_stellar_variability, + maximum_number_of_ensemble_comparisons_for_stellar_variability= + maximum_number_of_ensemble_comparisons_for_stellar_variability, + calibration_stars=vsp_comp_stars, + observed_filter=exotic_infoDict.get( + 'observed_filter', + exotic_infoDict.get('filter'), + ), + target_catalog_match=primary_target_catalog_match, + require_apparent_magnitudes=require_apparent_magnitudes, + use_exactly_the_comps_provided=use_exactly_the_comps_provided, + ) + else: + comparison_fit_search = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld, + pDict, + comparison_calibration, + psf_data, + aper_data, + tFlux, + psf_flux_data=psf_flux_source, + psf_noise_data=psf_noise_data if use_psf_photometry else None, + plot_time_range=full_plot_time_range, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + detrend_on_outoftransit_baseline=detrend_on_outoftransit_baseline, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, + pick_comparison_by_eebls_snr=pick_comparison_by_eebls_snr, + exit_at_first_qc_pass_solution=exit_at_first_qc_pass_solution, + final_fit_baseline_duration_multiplier=final_fit_baseline_duration_multiplier, + use_adaptive_apertures=use_adaptive_apertures, + adaptive_aperture_values=aperture_values, + adaptive_annulus_values=annulus_values, + fallback_sigma=sigma_display, + run_fast_ultranest_before_final_run=run_fast_ultranest_before_final_run, + run_final_fit_phase_residual_clip=run_final_fit_phase_residual_clip, + run_final_residual_rejection=run_final_residual_rejection, + save_dir=exotic_infoDict['save'], + planet_name=pDict['pName'], + observation_date=exotic_infoDict['date'], + use_ensemble_photometry_rather_than_single_comp= + use_ensemble_photometry_rather_than_single_comp, + maximum_number_of_ensemble_comparisons_for_transit= + maximum_number_of_ensemble_comparisons_for_transit, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=fallback_gain_e_per_adu, + use_exactly_the_comps_provided=use_exactly_the_comps_provided, + ) + if use_ensemble_photometry_for_stellar_variability and vsp_comp_stars: + log_info( + "Stellar-variability products only: selecting an independent calibrated " + "comparison-star ensemble of up to " + f"{maximum_number_of_ensemble_comparisons_for_stellar_variability} stars for " + "out-of-transit AID output. This ensemble is not used by the transit fit." + ) + stellar_variability_output_selection = select_stellar_variability_only_photometry( + times, + jd_times, + airmass, + pDict, + comparison_calibration, + psf_data, + aper_data, + tFlux, + psf_flux_data=psf_flux_source, + psf_noise_data=psf_noise_data if use_psf_photometry else None, + plot_time_range=full_plot_time_range, + use_adaptive_apertures=use_adaptive_apertures, + adaptive_aperture_values=aperture_values, + adaptive_annulus_values=annulus_values, + fallback_sigma=sigma_display, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=fallback_gain_e_per_adu, + use_ensemble_photometry=True, + maximum_number_of_ensemble_comparisons_for_stellar_variability= + maximum_number_of_ensemble_comparisons_for_stellar_variability, + calibration_stars=vsp_comp_stars, + observed_filter=exotic_infoDict.get( + 'observed_filter', + exotic_infoDict.get('filter'), + ), + target_catalog_match=primary_target_catalog_match, + require_apparent_magnitudes=require_apparent_magnitudes, + use_exactly_the_comps_provided=use_exactly_the_comps_provided, + ) + comparison_calibration['ranked_fit_comp_indices'] = [ + summary['comp_index'] for summary in comparison_fit_search['ranked_summaries'] + ] + comparison_calibration['fit_attempt_summaries'] = comparison_fit_search['attempts'] + + selected_attempt = comparison_fit_search['selected_result'] + fit_attempts = comparison_fit_search['attempts'] + if fit_attempts: + comparison_calibration['selected_fit_diagnostics'] = ( + selected_attempt['fit_diagnostics'] + if selected_attempt is not None + else fit_attempts[-1]['fit_diagnostics'] + ) + if selected_attempt is None or len(fit_attempts) > 1: + log_comparison_calibration_fit_attempt_summaries( + fit_attempts, + comparison_calibration['method_label'], + ) + + if selected_attempt is not None: + selected_comp_index = selected_attempt['comp_index'] + selected_ckey = selected_attempt['ckey'] + selected_is_ensemble = selected_comp_index is None + selected_comp_coords = ( + None + if selected_is_ensemble + else science_comp_stars[selected_comp_index] + ) + finder_entries = selected_comparison_finder_entries( + science_comp_stars, + comp_index=selected_comp_index, + ensemble_member_keys=selected_attempt.get('ensemble_member_keys'), + ) + selected_min_aperture = 0 if comparison_calibration['method'] == 'psf' else comparison_calibration['aper'] + selected_min_annulus = comparison_calibration['annulus'] + selected_a = None if comparison_calibration['method'] == 'psf' else comparison_calibration['a'] + selected_an = None if comparison_calibration['method'] == 'psf' else comparison_calibration['an'] + myfit = selected_attempt['fit'] + tFlux1 = selected_attempt['tflux_fit'] + cFlux1 = selected_attempt['cflux_fit'] + tFlux1_error = selected_attempt.get('tflux_fit_error') + cFlux1_error = selected_attempt.get('cflux_fit_error') + if tFlux1_error is None or np.shape(tFlux1_error) != np.shape(tFlux1): + tFlux1_error = tFlux1 ** 0.5 + if cFlux1_error is None or np.shape(cFlux1_error) != np.shape(cFlux1): + cFlux1_error = cFlux1 ** 0.5 + selected_source_indices = np.asarray( + selected_attempt.get('source_indices', np.arange(len(tFlux1), dtype=int)), + dtype=int, + ) + selected_attempt_label = selected_attempt.get('label', 'comparison candidate') + if stellar_variability_only and selected_is_ensemble: + selection_basis = ( + 'exact_stellar_variability_ensemble' + if use_exactly_the_comps_provided + else 'stellar_variability_ensemble' + ) + elif stellar_variability_only: + selection_basis = ( + 'exact_single_comparison' + if use_exactly_the_comps_provided + else 'stellar_variability_scatter' + ) + elif use_exactly_the_comps_provided and selected_is_ensemble: + selection_basis = 'exact_comparison_ensemble' + elif use_exactly_the_comps_provided: + selection_basis = 'exact_single_comparison' + elif selected_attempt.get('search_stopped_after_qc_pass', False): + selection_basis = 'first_qc_pass' + elif selected_attempt.get('search_stopped_after_promising_partial', False): + selection_basis = 'promising_partial' + elif selected_attempt.get('selected_despite_transit_qc', False): + selection_basis = 'comparison_field_qc_fallback' + elif selected_is_ensemble: + selection_basis = 'comparison_ensemble' + elif selected_comp_index == comparison_calibration['best_comp_index']: + selection_basis = 'comparison_field' + else: + selection_basis = 'comparison_field_retry' + if selection_basis == 'exact_stellar_variability_ensemble': + ensemble_members = selected_attempt.get('ensemble_member_keys') or [] + log_info( + "Stellar-variability-only comparison selection used the exact supplied " + f"ensemble with {len(ensemble_members)} member(s); no supplied comparison " + "was vetted out or replaced." + ) + elif selection_basis == 'stellar_variability_ensemble': + ensemble_members = selected_attempt.get('ensemble_member_keys') or [] + log_info( + "Stellar-variability-only comparison selection chose the default calibrated " + f"ensemble with {len(ensemble_members)} member(s) using " + f"{comparison_calibration['method_label']}; members are bright, unsaturated, " + "VSX-vetted, and passed the high-side catalog-error clip." + ) + elif selection_basis == 'exact_single_comparison': + log_info( + "Comparison selection used the one exact supplied comparison star; " + "no alternative comparison was evaluated." + ) + elif selection_basis == 'stellar_variability_scatter': + log_info( + "Stellar-variability-only comparison selection chose " + f"{selected_attempt_label} with {comparison_calibration['method_label']} " + "because it had the lowest out-of-transit target/reference scatter " + f"({format_residual_scatter(selected_attempt.get('selection_scatter', np.nan))})." + ) + elif selection_basis == 'first_qc_pass': + log_info( + "Comparison-star calibration target-fit selection chose " + f"{selected_attempt_label} with {comparison_calibration['method_label']} " + "because it was the first candidate to pass transit QC." + ) + elif selection_basis == 'promising_partial': + log_info( + "Comparison-star calibration target-fit selection chose " + f"{selected_attempt_label} with {comparison_calibration['method_label']} " + "because pre-UltraNest preflight and the candidate fit indicated a promising " + "partial-coverage MARGINAL solution." + ) + elif selection_basis == 'comparison_field_qc_fallback': + fallback_selection_metric = comparison_fit_search.get('selection_metric', 'ktmf') + if fallback_selection_metric == 'ktmf': + fallback_metric_value = format_ktmf_metric( + selected_attempt.get('ktmf_metric', np.nan) + ) + elif fallback_selection_metric in ('ktmf_scatter', 'ktmf_combined_quality'): + score_key = ( + 'combined_quality_ktmf_metric' + if fallback_selection_metric == 'ktmf_combined_quality' + else 'scatter_adjusted_ktmf_metric' + ) + score = selected_attempt.get(score_key, np.nan) + fallback_metric_value = ( + f"{score:.2f}" if np.isfinite(score) else "n/a" + ) + elif fallback_selection_metric == 'eebls_snr': + fallback_metric_value = format_eebls_snr( + selected_attempt.get('eebls_snr', np.nan) + ) + else: + fallback_metric_value = format_transit_delta_bic( + selected_attempt.get('transit_delta_bic', np.nan) + ) + log_info( + "Warning: all completed comparison-star target fits were rejected by transit QC; " + "continuing with the best available fit " + f"({selected_attempt_label}, " + f"{comparison_selection_metric_label(fallback_selection_metric)}=" + f"{fallback_metric_value}) so final outputs are still produced.", + warn=True, + ) + elif selection_basis == 'exact_comparison_ensemble': + log_info( + "Comparison-star calibration target-fit selection used every supplied " + "comparison star as one fixed ensemble because " + "'use_exactly_the_comps_provided' is enabled." + ) + elif selection_basis == 'comparison_ensemble': + log_info( + "Comparison-star calibration target-fit selection chose the comparison-star ensemble " + f"with {comparison_calibration['method_label']} because " + "'use_ensemble_photometry_rather_than_single_comp' is enabled." + ) + elif selection_basis == 'comparison_field_retry': + retry_count = selected_attempt['rank'] + log_info( + "Comparison-star calibration target-fit selection chose " + f"Comp {selected_comp_index + 1} with {comparison_calibration['method_label']} " + f"after evaluating {retry_count} better-ranked field-stability candidate(s); " + f"it delivered the best {comparison_selection_metric_label(comparison_fit_search['selection_metric'])} " + "among successful fits." + ) + + photometry_info.update(best_fit_lc=myfit, + comp_star_num=( + 'ensemble' if selected_is_ensemble else selected_comp_index + 1 + ), + comp_star_coords=selected_comp_coords, + finder_comparison_entries=finder_entries, + min_aperture=selected_min_aperture, + min_annulus=selected_min_annulus, + aperture_index=selected_a, + annulus_index=selected_an, + reuse_selected_full_reduction_fit=bool( + selected_attempt.get('full_reduction_applied', False) + and selected_attempt.get('fit') is not None + ), + selected_source_indices=selected_source_indices, + selected_fit_good_times=selected_attempt.get('good_times'), + selected_fit_good_flux=selected_attempt.get('good_flux'), + selected_fit_good_unc=selected_attempt.get('good_unc'), + selected_fit_good_airmass=selected_attempt.get('good_airmass'), + selected_fit_good_exposure_times_seconds=selected_attempt.get( + 'good_exposure_times_seconds' + ), + selected_fit_good_target_flux_error=selected_attempt.get('tflux_fit_error'), + selected_fit_good_comp_flux_error=selected_attempt.get('cflux_fit_error'), + selected_fit_duration_samples=selected_attempt.get('duration_samples'), + selected_fit_data_highres=selected_attempt.get('data_highres'), + selected_fit_final_output_dir=selected_attempt.get('final_output_dir'), + calibration_field_score=comparison_calibration['field_score'], + selection_basis=selection_basis, + stellar_variability_only=stellar_variability_only, + selection_metric=comparison_fit_search.get('selection_metric', 'ktmf'), + selected_comparison_selection_reason=selected_attempt.get('selection_reason'), + selected_comparison_attempt=compact_comparison_attempt_for_output(selected_attempt), + comparison_fit_attempt_summaries=[ + compact_comparison_attempt_for_output(attempt) + for attempt in comparison_fit_search.get('attempts', []) + ], + comparison_ktmf_metric=selected_attempt.get('ktmf_metric', np.nan), + selected_comparison_ktmf_contributions=selected_attempt.get('ktmf_contributions') or [], + comparison_eebls_snr=selected_attempt.get('eebls_snr', np.nan), + comparison_transit_delta_bic=selected_attempt.get('transit_delta_bic', np.nan), + selected_comparison_fit_point_count=selected_attempt.get('fit_point_count'), + selected_comparison_transit_qc_status=selected_attempt.get('transit_qc_status'), + selected_comparison_transit_qc_summary=selected_attempt.get('transit_qc_summary')) flux_values.update(flux_tar=tFlux1, flux_ref=cFlux1, - flux_unc_tar=tFlux1 ** 0.5, flux_unc_ref=cFlux1 ** 0.5) - - centroid_positions.update(x_targ=psf_data["target"][:, 0], y_targ=psf_data["target"][:, 1], - x_ref=psf_data[ckey][:, 0], y_ref=psf_data[ckey][:, 1]) - - if j in vsp_num: - ref_flux[j] = { - 'myfit': copy.deepcopy(myfit), - 'pos': exotic_infoDict['comp_stars'][j] - } - - log_info("\nComputing best comparison star, aperture, and sky annulus. Please wait.") - - # Aperture Photometry - for a, aper in enumerate(apers): - for an, annulus in enumerate(annuli): - tFlux = aper_data['target'][:, a, an] - ref_flux_opt, ref_flux_opt2, backtrack = False, False, True - temp_ref_flux = {i: None for i in vsp_num} - - # fit without a comparison star - myfit, tFlux1, cFlux1 = fit_lightcurve(times, tFlux, np.ones(tFlux.shape[0]), airmass, ld, pDict, jd_times) - - if myfit is not None: - for k in myfit.bounds.keys(): - log.debug(f" {k}: {myfit.parameters[k]:.6f}") - - log.debug("The Residual Standard Deviation is: " - f"{round(100 * myfit.residuals.std() / np.median(myfit.data), 6)}%") - log.debug(f"The Mean Squared Error is: {round(np.sum(myfit.residuals ** 2), 6)}\n") - - res_std = myfit.residuals.std() / np.median(myfit.data) - if photometry_info['min_std'] > res_std and myfit is not None: - ref_flux_opt = True - - photometry_info.update(best_fit_lc=copy.deepcopy(myfit), - comp_star_num=None, comp_star_coords=None, - min_std=res_std, min_aperture=-aper, min_annulus=annulus) - - flux_values.update(flux_tar=tFlux1, flux_ref=cFlux1, - flux_unc_tar=tFlux1 ** 0.5, flux_unc_ref=cFlux1 ** 0.5) - - centroid_positions.update(x_targ=psf_data["target"][:, 0], y_targ=psf_data["target"][:, 1], - x_ref=psf_data[ckey][:, 0], y_ref=psf_data[ckey][:, 1]) - - # try to fit data with comp star - for j in range(len(exotic_infoDict['comp_stars'])): - ckey = f"comp{j + 1}" - aper_mask = np.isfinite(aper_data[ckey][:, a, an]) - cFlux = aper_data[ckey][aper_mask][:, a, an] - - myfit, tFlux1, cFlux1 = fit_lightcurve(times[aper_mask], tFlux[aper_mask], cFlux, airmass[aper_mask], ld, pDict, jd_times[aper_mask]) - - if myfit is not None: - if j in vsp_num: - temp_ref_flux[j] = { - 'myfit': copy.deepcopy(myfit), - 'pos': exotic_infoDict['comp_stars'][j] + flux_unc_tar=tFlux1_error, flux_unc_ref=cFlux1_error) + + ref_centroid_x = np.full(selected_source_indices.shape, np.nan, dtype=float) + ref_centroid_y = np.full(selected_source_indices.shape, np.nan, dtype=float) + if selected_ckey in psf_data: + ref_centroid_x = psf_data[selected_ckey][selected_source_indices, 0] + ref_centroid_y = psf_data[selected_ckey][selected_source_indices, 1] + centroid_positions.update(x_targ=psf_data["target"][selected_source_indices, 0], + y_targ=psf_data["target"][selected_source_indices, 1], + x_ref=ref_centroid_x, + y_ref=ref_centroid_y) + + # A selected full-resolution retry can replace the original + # fit object after its raw photometry annotations were made. + # Reattach the final aligned flux arrays here so absolute + # stellar-variability magnitudes never have to be inferred + # from the normalized light curve. + annotate_stellar_variability_raw_photometry( + myfit, + tFlux1, + cFlux1, + target_flux_error=tFlux1_error, + comp_flux_error=cFlux1_error, + ) + myfit.differential_magnitude_reference_label = selected_attempt_label + + if selected_comp_index is not None: + ref_flux[selected_comp_index] = { + 'myfit': myfit, + 'pos': science_comp_stars[selected_comp_index] + } + + if vsp_num and not (stellar_variability_only and selected_is_ensemble): + if comparison_calibration['method'] == 'psf': + for j in vsp_num: + ckey = f"comp{j + 1}" + cFlux = psf_flux_series_from_rows(psf_flux_source[ckey]) + if stellar_variability_only: + vsp_fit, _ = build_stellar_variability_only_lightcurve_from_fluxes( + times, + tFlux, + cFlux, + airmass, + pDict, + jd_times=jd_times, + target_flux_error=psf_noise_data.get('target'), + comp_flux_error=psf_noise_data.get(ckey), + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=fallback_gain_e_per_adu, + comp_index=j, + comp_label=f"Comp {j + 1}", + comp_position=tracked_comparison_position(fortuitous_ensemble_stars, j), + method_label=comparison_calibration['method_label'], + plot_time_range=full_plot_time_range, + ) + else: + vsp_fit, _, _ = fit_lightcurve( + times, tFlux, cFlux, airmass, ld, pDict, jd_times, + target_flux_error=psf_noise_data.get('target'), + comp_flux_error=psf_noise_data.get(ckey), + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + plot_time_range=full_plot_time_range, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=fallback_gain_e_per_adu, + ) + if vsp_fit is None: + continue + ref_flux[j] = { + 'myfit': vsp_fit, + 'pos': tracked_comparison_position(fortuitous_ensemble_stars, j) } - - for k in myfit.bounds.keys(): - log.debug(f" {k}: {myfit.parameters[k]:.6f}") - - log.debug("The Residual Standard Deviation is: " - f"{round(100 * myfit.residuals.std() / np.median(myfit.data), 6)}%") - log.debug(f"The Mean Squared Error is: {round(np.sum(myfit.residuals ** 2), 6)}\n") - - res_std = myfit.residuals.std() / np.median(myfit.data) - if photometry_info['min_std'] > res_std and myfit is not None: # If the standard deviation is less than the previous min - ref_flux_opt2 = True - - photometry_info.update(best_fit_lc=copy.deepcopy(myfit), - comp_star_num=j + 1, - comp_star_coords=exotic_infoDict['comp_stars'][j], - min_std=res_std, min_aperture=aper, min_annulus=annulus) - - flux_values.update(flux_tar=tFlux1, flux_ref=cFlux1, - flux_unc_tar=tFlux1 ** 0.5, flux_unc_ref=cFlux1 ** 0.5) - - centroid_positions.update(x_targ=psf_data["target"][:, 0], y_targ=psf_data["target"][:, 1], - x_ref=psf_data[ckey][:, 0], y_ref=psf_data[ckey][:, 1]) - - if ref_flux_opt or ref_flux_opt2: - if j in vsp_num: + else: + best_a = comparison_calibration['a'] + best_an = comparison_calibration['an'] + best_target_flux = aper_data['target'][:, best_a, best_an] + best_target_flux_error = ( + aper_data['target_unc'][:, best_a, best_an] + if 'target_unc' in aper_data + else None + ) + for j in vsp_num: + ckey = f"comp{j + 1}" + aper_mask = np.isfinite(aper_data[ckey][:, best_a, best_an]) + cFlux = aper_data[ckey][aper_mask][:, best_a, best_an] + cFlux_error = ( + aper_data[f"{ckey}_unc"][aper_mask][:, best_a, best_an] + if f"{ckey}_unc" in aper_data + else None + ) + aper_exposure_times = ( + None + if exposure_times_seconds is None + else exposure_times_seconds[aper_mask] + ) + if stellar_variability_only: + vsp_fit, _ = build_stellar_variability_only_lightcurve_from_fluxes( + times[aper_mask], + best_target_flux[aper_mask], + cFlux, + airmass[aper_mask], + pDict, + jd_times=jd_times[aper_mask], + target_flux_error=( + None + if best_target_flux_error is None + else best_target_flux_error[aper_mask] + ), + comp_flux_error=cFlux_error, + exposure_times_seconds=aper_exposure_times, + gain_e_per_adu=fallback_gain_e_per_adu, + comp_index=j, + comp_label=f"Comp {j + 1}", + comp_position=tracked_comparison_position(fortuitous_ensemble_stars, j), + method_label=comparison_calibration['method_label'], + plot_time_range=full_plot_time_range, + ) + else: + vsp_fit, _, _ = fit_lightcurve( + times[aper_mask], best_target_flux[aper_mask], cFlux, + airmass[aper_mask], ld, pDict, jd_times[aper_mask], + target_flux_error=( + None if best_target_flux_error is None else best_target_flux_error[aper_mask] + ), + comp_flux_error=cFlux_error, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + plot_time_range=full_plot_time_range, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, + exposure_times_seconds=aper_exposure_times, + gain_e_per_adu=fallback_gain_e_per_adu, + ) + if vsp_fit is None: + continue ref_flux[j] = { - 'myfit': copy.deepcopy(myfit), - 'pos': exotic_infoDict['comp_stars'][j] + 'myfit': vsp_fit, + 'pos': tracked_comparison_position(fortuitous_ensemble_stars, j) } + else: + if fit_attempts: + failed_attempt = fit_attempts[-1] + failed_comp_index = failed_attempt['comp_index'] + failure_reason = failed_attempt['fit_diagnostics'].get( + 'failure_reason', + "the full comparison-star candidate reduction did not converge to a usable solution.", + ) + attempted_count = len(fit_attempts) + ranked_count = len(comparison_fit_search['ranked_summaries']) + failure_message = ( + "Comparison-star calibration exhausted " + f"{attempted_count}/{ranked_count} ranked comparison star(s) for " + f"{comparison_calibration['method_label']} without a usable fully reduced target fit " + f"(last attempt: Comp {failed_comp_index + 1}; reason: {failure_reason})." + ) + else: + failure_message = ( + "Comparison-star calibration did not produce any coverage-qualified " + "comparison stars to fully reduce against the target fit." + ) + if require_comp_star: + log_info(f"Error: {failure_message}", error=True) + return + log_info( + f"Warning: {failure_message} Falling back to raw target-flux aperture photometry " + "because require_comp_star is disabled.", + warn=True, + ) + + if photometry_info['best_fit_lc'] is None and not require_comp_star: + if not use_aperture_photometry or aper_data is None or apers is None or annuli is None: + log_info( + "Error: require_comp_star is disabled, but raw target-flux fallback requires usable " + "aperture photometry and no aperture grid is available.", + error=True, + ) + return + + log_info( + "\nNo usable comparison-star reduction was selected. Evaluating raw target-flux " + "aperture candidates because require_comp_star is disabled." + ) + raw_target_search = run_target_driven_photometry_search( + times, + jd_times, + airmass, + ld, + pDict, + [], + psf_data, + aper_data, + apers, + annuli, + sigma_display, + require_comp_star=False, + plot_time_range=full_plot_time_range, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + skip_low_comparison_coverage_rejection=True, + use_psf_photometry=False, + use_aperture_photometry=True, + multiprocess_lightcurve_fits=args.multiprocess_lightcurve_fits, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, + pick_comparison_by_eebls_snr=pick_comparison_by_eebls_snr, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=fallback_gain_e_per_adu, + psf_flux_data=psf_flux_source, + ) + if not apply_raw_target_photometry_selection( + raw_target_search, + photometry_info, + flux_values, + centroid_positions, + psf_data, + ): + candidate_summaries = raw_target_search.get('candidate_summaries', []) + if candidate_summaries: + log_target_fit_candidate_summaries(candidate_summaries) + log_info( + "Error: require_comp_star is disabled, but no raw target-flux aperture candidate " + "completed a usable reduction.", + error=True, + ) + return + log_info( + "Selected raw target-flux aperture photometry with no comparison star because " + "require_comp_star is disabled.", + warn=True, + ) + + update_photometry_adaptive_summary( + photometry_info, + use_adaptive_apertures, + aperture_values, + annulus_values, + psf_data['target'], + fallback_sigma=sigma, + ) - if backtrack: - for i, value in enumerate(temp_ref_flux.values()): - if value is not None and i != j: - ref_flux[i] = value - backtrack = False + if require_comp_star and photometry_info['comp_star_num'] is None: + log_info( + "Error: require_comp_star is enabled, but every evaluated comparison-star candidate " + "was rejected or failed to complete a usable full reduction. See the comparison-star " + "calibration fit diagnostics above for the per-candidate failure reasons.", + error=True, + ) + return log_info("\n\n*********************************************") + if np.isfinite(photometry_info['calibration_field_score']): + log_info(f"Comparison-Star Field Score: {round(photometry_info['calibration_field_score'] * 100, 4)}%") + summary_min_aperture = photometry_info.get('min_aperture') + if photometry_info.get('comp_star_num') is not None or ( + summary_min_aperture is not None and summary_min_aperture < 0 + ): + log_info( + "Comparison Selection Metric: " + f"{comparison_selection_metric_label(photometry_info.get('selection_metric', 'ktmf'))}" + ) + if np.isfinite(photometry_info.get('comparison_ktmf_metric', np.nan)): + log_info(f"Selected Comparison KTMF: {photometry_info['comparison_ktmf_metric']:.2f} / 5.00") + if np.isfinite(photometry_info.get('comparison_eebls_snr', np.nan)): + log_info(f"Selected Comparison EEBLS SNR: {photometry_info['comparison_eebls_snr']:.2f}") + if np.isfinite(photometry_info.get('comparison_transit_delta_bic', np.nan)): + log_info( + "Selected Comparison Transit Delta BIC: " + f"{photometry_info['comparison_transit_delta_bic']:.2f}" + ) + selected_method_label = selected_photometry_method_label(photometry_info) + display_aperture, display_annulus = reported_photometry_aperture_radii(photometry_info) + adaptive_summary = photometry_info.get('adaptive_summary') + comparison_star_log_label = ( + "Stellar Variability Reference Star" + if stellar_variability_only + else "Transit Fit Comparison Star" + ) if photometry_info['min_aperture'] == 0: # psf - log_info(f"Best Comparison Star: #{photometry_info['comp_star_num']}") - log_info(f"Minimum Residual Scatter: {round(photometry_info['min_std'] * 100, 4)}%") + if photometry_info.get('comp_star_num') == 'ensemble': + log_info(f"{comparison_star_log_label}: ensemble") + else: + log_info(f"{comparison_star_log_label}: #{photometry_info['comp_star_num']}") log_info("Optimal Method: PSF photometry") elif photometry_info['min_aperture'] < 0: # no comp star - log_info("Best Comparison Star: None") - log_info(f"Minimum Residual Scatter: {round(photometry_info['min_std'] * 100, 4)}%") - log_info(f"Optimal Aperture: {abs(np.round(photometry_info['min_aperture'], 2))}") - log_info(f"Optimal Annulus: {np.round(photometry_info['min_annulus'], 2)}") + log_info(f"{comparison_star_log_label}: None") + if adaptive_summary is not None: + log_info(f"Optimal Aperture: {abs(display_aperture):.2f} +/- {adaptive_summary['aperture_std']:.2f} px") + log_info(f"Optimal Annulus: {display_annulus:.2f} +/- {adaptive_summary['annulus_std']:.2f} px") + log_info(f"Adaptive Aperture Scale: {adaptive_summary['aperture_sigma']:.2f} sigma") + log_info(f"Adaptive Annulus Scale: {adaptive_summary['annulus_sigma']:.2f} sigma") + log_info(f"Aperture Range: {adaptive_summary['aperture_min']:.2f} to {adaptive_summary['aperture_max']:.2f} px") + log_info(f"Annulus Range: {adaptive_summary['annulus_min']:.2f} to {adaptive_summary['annulus_max']:.2f} px") + else: + log_info(f"Optimal Aperture: {abs(np.round(display_aperture, 2))}") + log_info(f"Optimal Annulus: {np.round(display_annulus, 2)}") else: - log_info(f"Best Comparison Star: #{photometry_info['comp_star_num']}") - log_info(f"Minimum Residual Scatter: {round(photometry_info['min_std'] * 100, 4)}%") - log_info(f"Optimal Aperture: {np.round(photometry_info['min_aperture'], 2)}") - log_info(f"Optimal Annulus: {np.round(photometry_info['min_annulus'], 2)}") + if photometry_info.get('comp_star_num') == 'ensemble': + log_info(f"{comparison_star_log_label}: ensemble") + else: + log_info(f"{comparison_star_log_label}: #{photometry_info['comp_star_num']}") + if adaptive_summary is not None: + log_info(f"Optimal Aperture: {display_aperture:.2f} +/- {adaptive_summary['aperture_std']:.2f} px") + log_info(f"Optimal Annulus: {display_annulus:.2f} +/- {adaptive_summary['annulus_std']:.2f} px") + log_info(f"Adaptive Aperture Scale: {adaptive_summary['aperture_sigma']:.2f} sigma") + log_info(f"Adaptive Annulus Scale: {adaptive_summary['annulus_sigma']:.2f} sigma") + log_info(f"Aperture Range: {adaptive_summary['aperture_min']:.2f} to {adaptive_summary['aperture_max']:.2f} px") + log_info(f"Annulus Range: {adaptive_summary['annulus_min']:.2f} to {adaptive_summary['annulus_max']:.2f} px") + else: + log_info(f"Optimal Aperture: {np.round(display_aperture, 2)}") + log_info(f"Optimal Annulus: {np.round(display_annulus, 2)}") log_info("*********************************************\n") + reduction_stage_timer.checkpoint("Comparison calibration and target light-curve selection") best_fit_lc = photometry_info['best_fit_lc'] bestCompStar = photometry_info['comp_star_num'] comp_coords = photometry_info['comp_star_coords'] + log_lightcurve_filter_diagnostics( + getattr(best_fit_lc, 'frame_filter_diagnostics', []), + header="Selected lightcurve frame rejections during target fitting", + ) + try: + selected_photometry_debug_path = save_selected_photometry_debug_series( + exotic_infoDict['save'], + pDict['pName'], + exotic_infoDict['date'], + best_fit_lc, + ) + if selected_photometry_debug_path is not None: + log_info( + f"Saved selected raw target/reference ratio diagnostics to " + f"{selected_photometry_debug_path}." + ) + except Exception as e: + log_info( + f"Warning: Could not save selected raw target/reference ratio diagnostics ({e}).", + warn=True, + ) + + if fit_every_comparison_candidate and stellar_variability_only: + log_info( + "Skipping fit_lightcurve_to_every_comparison_candidate because " + "stellar-variability-only mode does not fit transit models." + ) + + if fit_every_comparison_candidate and not stellar_variability_only and science_comp_stars: + candidate_fit_summaries = fit_lightcurve_to_every_comparison_candidate( + times, + jd_times, + airmass, + ld, + pDict, + science_comp_stars, + psf_data, + aper_data, + photometry_info, + plot_time_range=full_plot_time_range, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + skip_low_comparison_coverage_rejection=skip_low_comp_coverage_rejection, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + use_eebls_to_initialize_tmid_and_bounds=use_eebls_tmid_initializer, + psf_flux_data=psf_flux_source, + psf_noise_data=psf_noise_data if use_psf_photometry else None, + exposure_times_seconds=exposure_times_seconds, + gain_e_per_adu=fallback_gain_e_per_adu, + ) + saved_candidate_fit_count = sum(1 for summary in candidate_fit_summaries if summary['fit'] is not None) + failed_candidate_fit_count = len(candidate_fit_summaries) - saved_candidate_fit_count + if candidate_fit_summaries: + log_comparison_candidate_fit_summaries(candidate_fit_summaries, photometry_info) + try: + plot_comp_star_candidate_lightcurve_fits( + candidate_fit_summaries, + pDict['pName'], + exotic_infoDict['save'], + exotic_infoDict['date'], + selected_method_label, + ) + log_info( + f"Saved {saved_candidate_fit_count} comparison-candidate lightcurve fit plot(s) to working_artifacts/." + ) + if failed_candidate_fit_count: + log_info( + f"Skipped {failed_candidate_fit_count} comparison candidate(s) that did not yield a usable lightcurve fit." + ) + except Exception as e: + log_info(f"Warning: Could not save comparison-candidate lightcurve plots ({e}).", warn=True) # save psf_data to disk for best comparison star - if bestCompStar: - np.savetxt(Path(exotic_infoDict['save']) / "temp" / "psf_data_comp.txt", psf_data[f"comp{bestCompStar}"], + if isinstance(bestCompStar, int): + np.savetxt(working_artifacts_dir / "psf_data_comp.txt", psf_data[f"comp{bestCompStar}"], header="#x_centroid, y_centroid, amplitude, sigma_x, sigma_y, rotation offset", fmt="%.6f") + reuse_selected_full_reduction_fit = bool( + photometry_info.get('reuse_selected_full_reduction_fit', False) + and best_fit_lc is not None + ) + psf_selection_indices = np.asarray( + photometry_info.get('selected_source_indices', np.arange(len(best_fit_lc.time))), + dtype=int, + ) + if psf_selection_indices.shape[0] != len(best_fit_lc.time): + psf_selection_indices = np.arange(len(best_fit_lc.time), dtype=int) + # sigma clip - si = np.argsort(best_fit_lc.time) - dt = np.mean(np.diff(np.sort(best_fit_lc.time))) - ndt = int(30. / 24. / 60. / dt) * 2 + 1 # ~30 minutes - gi = ~sigma_clip(best_fit_lc.data[si], sigma=3, dt=ndt) # good indexs - - # Calculate the proper timeseries uncertainties from the residuals of the out-of-transit data - OOT = (best_fit_lc.transit == 1) # find out-of-transit portion of the lightcurve - - if sum(OOT) <= 1: - OOTscatter = np.std(best_fit_lc.residuals) - goodNormUnc = OOTscatter * best_fit_lc.airmass_model - goodNormUnc = goodNormUnc / np.nanmedian(best_fit_lc.data) - goodFluxes = best_fit_lc.data / np.nanmedian(best_fit_lc.data) + if reuse_selected_full_reduction_fit: + log_info( + "Reusing the selected comparison-star full-reduction ultranest fit; " + "skipping duplicate selected-only final-fit clipping." + ) + si = np.arange(len(best_fit_lc.time), dtype=int) + time_clip_mask = np.zeros(len(best_fit_lc.time), dtype=bool) + phase_clip_mask = np.zeros_like(time_clip_mask, dtype=bool) + adaptive_clip_mask = np.zeros_like(time_clip_mask, dtype=bool) else: - OOTscatter = np.std((best_fit_lc.data / best_fit_lc.airmass_model)[OOT]) # calculate the scatter in the data - goodNormUnc = OOTscatter * best_fit_lc.airmass_model # scale this scatter back up by the airmass model and then adopt these as the uncertainties - goodNormUnc = goodNormUnc / np.nanmedian(best_fit_lc.data[OOT]) - goodFluxes = best_fit_lc.data / np.nanmedian(best_fit_lc.data[OOT]) + si = np.argsort(best_fit_lc.time) + dt = np.mean(np.diff(np.sort(best_fit_lc.time))) + ndt = int(30. / 24. / 60. / dt) * 2 + 1 # ~30 minutes + time_clip_mask = sigma_clip(best_fit_lc.data[si], sigma=3, dt=ndt, times=best_fit_lc.time[si]) + phase_clip_mask = np.zeros_like(time_clip_mask, dtype=bool) + if ( + run_final_fit_phase_residual_clip + and hasattr(best_fit_lc, 'residuals') + and hasattr(best_fit_lc, 'phase') + ): + phase_clip_mask = phase_bin_sigma_clip(best_fit_lc.residuals[si], best_fit_lc.phase[si], sigma=3, bins=10) + adaptive_clip_mask = np.zeros_like(time_clip_mask, dtype=bool) + if use_adaptive_apertures and adaptive_summary is not None: + sorted_apertures = np.asarray(adaptive_summary['aperture_series'], dtype=float)[si] + sorted_annuli = np.asarray(adaptive_summary['annulus_series'], dtype=float)[si] + retained_mask = adaptive_aperture_outlier_mask(sorted_apertures[~time_clip_mask & ~phase_clip_mask], + sorted_annuli[~time_clip_mask & ~phase_clip_mask]) + adaptive_clip_mask[~time_clip_mask & ~phase_clip_mask] = retained_mask + gi = ~(time_clip_mask | phase_clip_mask | adaptive_clip_mask) # good indexs + prefinal_filter_diagnostics = [ + build_time_rejection_diagnostic( + "Final-fit time sigma clip", + np.asarray(best_fit_lc.time, dtype=float)[si], + ~time_clip_mask, + note="Dropped time-series outliers before the final fit.", + ), + build_time_rejection_diagnostic( + "Final-fit phase residual clip", + np.asarray(best_fit_lc.time, dtype=float)[si], + ~phase_clip_mask, + note="Dropped phase-binned residual outliers before the final fit.", + ), + build_time_rejection_diagnostic( + "Final-fit adaptive-aperture clip", + np.asarray(best_fit_lc.time, dtype=float)[si], + ~adaptive_clip_mask, + note="Dropped adaptive-aperture radius outliers before the final fit.", + ), + ] + phase_clip_removed = np.count_nonzero(phase_clip_mask & ~time_clip_mask) + if phase_clip_removed: + log_info(f"Removed {phase_clip_removed} phase-binned residual outlier(s) before final fit.") + adaptive_clip_removed = np.count_nonzero(adaptive_clip_mask) + if adaptive_clip_removed: + log_info(f"Removed {adaptive_clip_removed} adaptive-aperture radius outlier(s) before final fit.") if np.isnan(best_fit_lc.data).all(): log_info("Error: No valid photometry data found.", error=True) return - best_fit_lc.time = best_fit_lc.time[si][gi] - best_fit_lc.data = best_fit_lc.data[si][gi] - best_fit_lc.airmass = best_fit_lc.airmass[si][gi] - best_fit_lc.transit = best_fit_lc.transit[si][gi] - best_fit_lc.jd_times = best_fit_lc.jd_times[si][gi] + apply_lightcurve_mask(best_fit_lc, gi, sort_index=si) goodTimes = best_fit_lc.time - goodFluxes = goodFluxes[si][gi] - goodNormUnc = goodNormUnc[si][gi] goodAirmasses = best_fit_lc.airmass + goodExposureTimes = None + fit_exposure_days = getattr(best_fit_lc, 'exposure_times_days', None) + if fit_exposure_days is not None: + fit_exposure_days = np.asarray(fit_exposure_days, dtype=float) + if fit_exposure_days.shape == np.shape(goodTimes): + goodExposureTimes = fit_exposure_days * 86400.0 + selected_good_exposure_times = photometry_info.get('selected_fit_good_exposure_times_seconds') + if ( + selected_good_exposure_times is not None + and np.shape(selected_good_exposure_times) == np.shape(goodTimes) + ): + goodExposureTimes = np.asarray(selected_good_exposure_times, dtype=float) + + if reuse_selected_full_reduction_fit: + selected_good_flux = photometry_info.get('selected_fit_good_flux') + selected_good_unc = photometry_info.get('selected_fit_good_unc') + if selected_good_flux is not None and np.shape(selected_good_flux) == np.shape(goodTimes): + goodFluxes = np.asarray(selected_good_flux, dtype=float) + else: + goodFluxes = np.asarray(best_fit_lc.detrended, dtype=float) + if selected_good_unc is not None and np.shape(selected_good_unc) == np.shape(goodTimes): + goodNormUnc = np.asarray(selected_good_unc, dtype=float) + else: + goodNormUnc = np.asarray(best_fit_lc.detrendederr, dtype=float) + else: + final_fit_series = prepare_final_fit_lightcurve_series(best_fit_lc) + if not final_fit_series.get('applied'): + log_info( + f"Warning: {final_fit_series.get('note', 'could not prepare the final-fit light curve from the selected fit.')} " + "Falling back to the current detrended light curve arrays.", + warn=True, + ) + goodFluxes = np.asarray(best_fit_lc.detrended, dtype=float) + goodNormUnc = np.asarray(best_fit_lc.detrendederr, dtype=float) + else: + log_info(final_fit_series['note']) + goodFluxes = np.asarray(final_fit_series['flux'], dtype=float) + goodNormUnc = np.asarray(final_fit_series['unc'], dtype=float) centroid_positions.update(x_targ=centroid_positions['x_targ'][si][gi], y_targ=centroid_positions['y_targ'][si][gi], @@ -2516,6 +36133,57 @@ def main(): flux_unc_tar=flux_values['flux_unc_tar'][si][gi], flux_unc_ref=flux_values['flux_unc_ref'][si][gi]) + relative_flux_mask = relative_flux_filter_mask(goodFluxes) + prefinal_filter_diagnostics.append(build_time_rejection_diagnostic( + "Final relative-flux validity filter", + goodTimes, + relative_flux_mask, + note="Dropped non-finite or non-positive normalized flux values before the final fit.", + )) + if np.count_nonzero(relative_flux_mask) == 0: + log_info( + "Error: No valid photometry data found after removing non-finite or non-positive relative flux values.", + error=True, + ) + return + + log_lightcurve_filter_diagnostics( + prefinal_filter_diagnostics, + header="Selected lightcurve frame rejections before the final fit", + ) + apply_lightcurve_mask(best_fit_lc, relative_flux_mask) + + goodTimes = goodTimes[relative_flux_mask] + goodFluxes = goodFluxes[relative_flux_mask] + goodNormUnc = goodNormUnc[relative_flux_mask] + goodAirmasses = goodAirmasses[relative_flux_mask] + if goodExposureTimes is not None: + goodExposureTimes = goodExposureTimes[relative_flux_mask] + + centroid_positions.update(x_targ=centroid_positions['x_targ'][relative_flux_mask], + y_targ=centroid_positions['y_targ'][relative_flux_mask], + x_ref=centroid_positions['x_ref'][relative_flux_mask], + y_ref=centroid_positions['y_ref'][relative_flux_mask]) + + flux_values.update(flux_tar=flux_values['flux_tar'][relative_flux_mask], + flux_ref=flux_values['flux_ref'][relative_flux_mask], + flux_unc_tar=flux_values['flux_unc_tar'][relative_flux_mask], + flux_unc_ref=flux_values['flux_unc_ref'][relative_flux_mask]) + + psf_selection_indices = psf_selection_indices[si][gi][relative_flux_mask] + obs_stats_sort_index = psf_selection_indices + obs_stats_keep_mask = np.ones(psf_selection_indices.shape[0], dtype=bool) + + update_photometry_adaptive_summary( + photometry_info, + use_adaptive_apertures, + aperture_values, + annulus_values, + psf_data['target'][psf_selection_indices], + fallback_sigma=sigma, + ) + display_aperture, display_annulus = reported_photometry_aperture_radii(photometry_info) + if photometry_info['min_aperture'] == 0: opt_method = "PSF" @@ -2525,23 +36193,91 @@ def main(): min_annulus_fov = float(15 * stdev_fov.mean()) else: opt_method = "Aperture" - min_aper_fov = float(photometry_info['min_aperture']) - min_annulus_fov = float(photometry_info['min_annulus']) - - plot_fov(photometry_info['min_aperture'], photometry_info['min_annulus'], sigma, - centroid_positions['x_targ'][0], centroid_positions['y_targ'][0], - centroid_positions['x_ref'][0], centroid_positions['y_ref'][0], - firstImage, img_scale_str, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date'], opt_method, min_aper_fov, min_annulus_fov) + min_aper_fov = float(display_aperture) + min_annulus_fov = float(display_annulus) + + fov_aperture = min_aper_fov if opt_method == "PSF" else float(display_aperture) + fov_annulus = min_annulus_fov if opt_method == "PSF" else float(display_annulus) + fov_sky_geometry = resolve_sky_annulus_geometry( + fov_aperture, + fov_annulus, + psf_sigma=sigma_display, + ) - plot_centroids(centroid_positions['x_targ'], centroid_positions['y_targ'], - centroid_positions['x_ref'], centroid_positions['y_ref'], - goodTimes, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date']) + reference_centroid_available = ( + np.isfinite(centroid_positions['x_ref'][0]) + and np.isfinite(centroid_positions['y_ref'][0]) + ) + finder_entries = list(photometry_info.get('finder_comparison_entries') or []) + if not finder_entries and reference_centroid_available: + finder_entries = [{ + 'label': 'Comp Star', + 'position': [ + float(centroid_positions['x_ref'][0]), + float(centroid_positions['y_ref'][0]), + ], + }] + if finder_entries: + firstImage = ensure_first_reduction_image_for_fov( + firstImage, + inputfiles[0], + generalDark, + generalBias, + generalFlat, + demosaic_fmt, + demosaic_out, + demosaic_mult, + bad_pixel_reference=bad_pixel_reference, + ) + finder_positions = [entry['position'] for entry in finder_entries] + finder_labels = [entry['label'] for entry in finder_entries] + finder_target_position = [ + float(centroid_positions['x_targ'][0]), + float(centroid_positions['y_targ'][0]), + ] + first_target_centroid = np.asarray(psf_data['target'][0, :2], dtype=float) + if np.all(np.isfinite(first_target_centroid)): + finder_target_position = first_target_centroid.tolist() + plot_fov(fov_aperture, fov_annulus, sigma_display, + finder_target_position[0], finder_target_position[1], + finder_positions[0][0], finder_positions[0][1], + firstImage, img_scale_str, pDict['pName'], exotic_infoDict['save'], + exotic_infoDict['date'], opt_method, min_aper_fov, min_annulus_fov, + sky_inner_radius=fov_sky_geometry['inner_radius'], + sky_outer_radius=fov_sky_geometry['outer_radius'], + comparison_positions=finder_positions, + comparison_labels=finder_labels) + + if reference_centroid_available: + plot_centroids(centroid_positions['x_targ'], centroid_positions['y_targ'], + centroid_positions['x_ref'], centroid_positions['y_ref'], + goodTimes, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date']) + else: + log_info( + "Skipping reference-star centroid plots because the selected reference does not " + "have one single-star centroid series." + ) plot_flux(goodTimes, flux_values['flux_tar'], flux_values['flux_unc_tar'], flux_values['flux_ref'], flux_values['flux_unc_ref'], goodFluxes, goodNormUnc, goodAirmasses, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date']) + adaptive_summary = photometry_info.get('adaptive_summary') + if adaptive_summary is not None: + plot_adaptive_aperture_diagnostics( + goodTimes, + adaptive_summary['aperture_series'], + adaptive_summary['annulus_series'], + adaptive_summary['fwhm_series'], + goodAirmasses, + pDict['pName'], + exotic_infoDict['save'], + exotic_infoDict['date'], + adaptive_summary['aperture_sigma'], + adaptive_summary['annulus_sigma'], + ) + # TODO: convert the exoplanet archive mid transit time to bjd - need to take into account observatory location listed in Exoplanet Archive # tMidtoC = astropy.time.Time(timeMidTransit, format='jd', scale='utc') # forPhaseResult = JDUTC_to_BJDTDB(tMidtoC, ra=raDeg, dec=decDeg, lat=lati, longi=longit, alt=2000) @@ -2558,21 +36294,80 @@ def main(): # Calculate the standard deviation of the normalized flux values # standardDev1 = np.std(goodFluxes) - if vsp_comp_stars: - if not bestCompStar: - vsp_params = stellar_variability(ref_flux, best_fit_lc, exotic_infoDict['comp_stars'], - vsp_comp_stars, vsp_num, None, exotic_infoDict['save'], - pDict['sName']) - else: - vsp_params = stellar_variability(ref_flux, best_fit_lc, exotic_infoDict['comp_stars'], - vsp_comp_stars, vsp_num, bestCompStar - 1, exotic_infoDict['save'], - pDict['sName']) + if stellar_variability_output_selection is not None: + try: + vsp_params = build_stellar_variability_params_from_photometry_selection( + stellar_variability_output_selection, + vsp_comp_stars, + exotic_infoDict['save'], + pDict['sName'], + observed_filter=exotic_infoDict.get( + 'observed_filter', + exotic_infoDict.get('filter'), + ), + observation_date=exotic_infoDict.get('date'), + target_metadata={ + 'name': pDict.get('sName'), + 'ra_deg': None if ra_dec_tar is None else ra_dec_tar[0], + 'dec_deg': None if ra_dec_tar is None else ra_dec_tar[1], + 'pixel_position': [exotic_UIprevTPX, exotic_UIprevTPY], + }, + ) + except Exception as exc: + log_info( + "Warning: could not create calibrated ensemble stellar-variability " + f"AID rows ({describe_retry_exception(exc)}).", + warn=True, + ) + vsp_params = [] + + if not vsp_params and stellar_variability_only and bestCompStar == 'ensemble': + vsp_params = build_stellar_variability_ensemble_params_from_fit( + best_fit_lc, + exotic_infoDict['save'], + pDict['sName'], + observed_filter=exotic_infoDict.get( + 'observed_filter', + exotic_infoDict.get('filter'), + ), + observation_date=exotic_infoDict.get('date'), + target_metadata={ + 'name': pDict.get('sName'), + 'ra_deg': None if ra_dec_tar is None else ra_dec_tar[0], + 'dec_deg': None if ra_dec_tar is None else ra_dec_tar[1], + 'pixel_position': [exotic_UIprevTPX, exotic_UIprevTPY], + }, + ) + elif not vsp_params and vsp_comp_stars: + if isinstance(bestCompStar, int): + vsp_params = stellar_variability(ref_flux, best_fit_lc, fortuitous_ensemble_stars, + vsp_comp_stars, vsp_num, bestCompStar - 1, exotic_infoDict['save'], + pDict['sName'], + observed_filter=exotic_infoDict.get('observed_filter', + exotic_infoDict.get('filter')), + observation_date=exotic_infoDict.get('date'), + comp_ra_dec=ra_dec_wcs[:len(fortuitous_ensemble_stars)], + field_catalog=nextastro_field_catalog, + reference_image=reference_image, + wcs_file=wcs_file, + catalog_match_radius_arcsec= + photometry_catalog_match_radius_arcsec) + elif stellar_variability_output_selection is None: + log_info( + "Skipping AID magnitude output because no reference comparison star was selected.", + warn=True, + ) log_info("\n\nOutput File Saved") else: goodTimes, goodFluxes, goodNormUnc, goodAirmasses = [], [], [], [] bestCompStar, comp_coords = None, None - ld, ld0, ld1, ld2, ld3 = get_ld_values(pDict, exotic_infoDict) + exotic_infoDict.setdefault('observed_filter', exotic_infoDict.get('filter')) + ld, ld0, ld1, ld2, ld3 = get_ld_values( + pDict, + exotic_infoDict, + non_interactive_run=args.non_interactive_run, + ) with exotic_infoDict['prered_file'].open('r') as f: for processed_data in f: @@ -2591,6 +36386,14 @@ def main(): goodAirmasses = np.array(goodAirmasses) if exotic_infoDict['file_time'] != 'BJD_TDB': + missing_location = [ + label for key, label in (('long', 'longitude'), ('lat', 'latitude'), ('elev', 'elevation')) + if exotic_infoDict.get(key) is None + ] + if missing_location: + log_info("Error: Longitude, latitude, and elevation are required to convert " + f"pre-reduced {exotic_infoDict['file_time']} timestamps to BJD_TDB.", error=True) + return time_offset = 2400000.5 if exotic_infoDict['file_time'] == 'MJD_UTC' else 0.0 goodTimes = convert_jd_to_bjd([time_ + time_offset for time_ in goodTimes], pDict, exotic_infoDict) @@ -2598,6 +36401,36 @@ def main(): print("check flux convert") goodFluxes, goodNormUnc = flux_conversion(goodFluxes, goodNormUnc, exotic_infoDict['file_units']) + relative_flux_mask = relative_flux_filter_mask(goodFluxes) + if np.count_nonzero(relative_flux_mask) == 0: + log_info( + "Error: No valid photometry data found after removing non-finite or non-positive relative flux values.", + error=True, + ) + return + + goodTimes = goodTimes[relative_flux_mask] + goodFluxes = goodFluxes[relative_flux_mask] + goodNormUnc = goodNormUnc[relative_flux_mask] + goodAirmasses = goodAirmasses[relative_flux_mask] + try: + prereduced_exposure = float(exotic_infoDict.get('exposure', np.nan)) + except (TypeError, ValueError): + prereduced_exposure = np.nan + goodExposureTimes = ( + np.full(goodTimes.shape, prereduced_exposure, dtype=float) + if np.isfinite(prereduced_exposure) and prereduced_exposure > 0 + else None + ) + goodFluxes, goodNormUnc, _ = normalize_flux_series_to_approximate_unity( + goodFluxes, + goodNormUnc, + ) + finite_plot_times = goodTimes[np.isfinite(goodTimes)] + full_plot_time_range = None + if finite_plot_times.size: + full_plot_time_range = (float(np.min(finite_plot_times)), float(np.max(finite_plot_times))) + # for k in myfit.bounds.keys(): # print(f"{myfit.parameters[k]:.6f} +- {myfit.errors[k]}") @@ -2606,9 +36439,191 @@ def main(): return log_info("\n") - log_info("****************************************") - log_info("Fitting a Light Curve Model to Your Data") - log_info("****************************************\n") + if stellar_variability_only: + log_info("****************************************") + log_info("Preparing Stellar Variability Light Curve") + log_info("****************************************\n") + else: + log_info("****************************************") + log_info("Fitting a Light Curve Model to Your Data") + log_info("****************************************\n") + + reuse_selected_final_model = bool( + fitsortext == 1 + and photometry_info.get('reuse_selected_full_reduction_fit', False) + and photometry_info.get('best_fit_lc') is not None + ) + + if stellar_variability_only: + if reuse_selected_final_model: + myfit = photometry_info['best_fit_lc'] + goodTimes = np.asarray(getattr(myfit, 'time', goodTimes), dtype=float) + goodAirmasses = np.asarray(getattr(myfit, 'airmass', goodAirmasses), dtype=float) + reused_flux = photometry_info.get('selected_fit_good_flux') + reused_unc = photometry_info.get('selected_fit_good_unc') + if reused_flux is not None and np.shape(reused_flux) == np.shape(goodTimes): + goodFluxes = np.asarray(reused_flux, dtype=float) + else: + goodFluxes = np.asarray(getattr(myfit, 'detrended', goodFluxes), dtype=float) + if reused_unc is not None and np.shape(reused_unc) == np.shape(goodTimes): + goodNormUnc = np.asarray(reused_unc, dtype=float) + else: + goodNormUnc = np.asarray(getattr(myfit, 'detrendederr', goodNormUnc), dtype=float) + log_info( + "Using the selected comparison-star stellar-variability light curve for final outputs; " + "no transit model fit is being run." + ) + else: + prepared_variability = { + 'applied': True, + 'time': np.asarray(goodTimes, dtype=float), + 'flux': np.asarray(goodFluxes, dtype=float), + 'unc': np.asarray(goodNormUnc, dtype=float), + 'airmass': np.asarray(goodAirmasses, dtype=float), + 'jd_time': np.asarray(goodTimes, dtype=float), + 'exposure_time_seconds': goodExposureTimes, + 'target_flux': np.asarray(goodFluxes, dtype=float), + 'comp_flux': np.ones(np.shape(goodFluxes), dtype=float), + 'target_flux_error': np.asarray(goodNormUnc, dtype=float), + 'comp_flux_error': np.full(np.shape(goodFluxes), np.nan, dtype=float), + 'source_indices': np.arange(np.shape(goodFluxes)[0], dtype=int), + } + myfit = build_stellar_variability_only_lightcurve( + prepared_variability, + pDict, + method_label="pre-reduced light curve", + plot_time_range=full_plot_time_range, + ) + if myfit is None: + log_info( + "Error: stellar-variability-only mode could not build a usable " + "out-of-transit light curve from the supplied pre-reduced data.", + error=True, + ) + return + + if np.any(np.isfinite(myfit.time)): + times = np.linspace(np.nanmin(myfit.time), np.nanmax(myfit.time), 1000) + else: + times = np.array([], dtype=float) + data_highres = np.ones(times.shape, dtype=float) + exclusion = getattr(myfit, 'stellar_variability_transit_exclusion', {}) or {} + duration = exclusion.get('duration_days', np.nan) + durs = [duration] if np.isfinite(duration) else [] + if vsp_params: + try: + myfit.stellar_variability_params = vsp_params + myfit.stellar_variability_target_name = pDict.get('sName', pDict.get('pName')) + myfit.stellar_variability_reference_label = vsp_params[0].get('cname') + except Exception: + pass + + plot_final_lightcurve( + myfit, + data_highres, + pDict['pName'], + exotic_infoDict['save'], + exotic_infoDict['date'], + observed_filter=exotic_infoDict.get('observed_filter', exotic_infoDict.get('filter')), + ) + + if fitsortext == 1: + observing_background_series = build_observing_background_series( + psf_data, + aper_data, + photometry_info, + len(science_comp_stars), + ) + plot_obs_stats(myfit, science_comp_stars, psf_data, obs_stats_sort_index, + obs_stats_keep_mask, pDict['pName'], + exotic_infoDict['save'], exotic_infoDict['date'], + relative_flux_mask=None, + background_series=observing_background_series) + + reduction_stage_timer.checkpoint("Final stellar-variability light curve, plots, and diagnostics") + + log_info("\n*********************************************************") + log_info("FINAL STELLAR VARIABILITY ANALYSIS\n") + log_info(" Analysis Mode: stellar variability only") + log_info(" Transit model fitting: skipped") + scatter = getattr(myfit, 'stellar_variability_scatter', np.nan) + if np.isfinite(scatter): + log_info(f" Out-of-transit scatter: {format_residual_scatter(scatter)}") + log_info(f" Light-curve point count: {len(myfit.time)}") + rejected_points = exclusion.get('rejected_point_count', 0) + log_info(f" Predicted in-transit points excluded: {rejected_points}") + if np.isfinite(duration): + log_info(f" Excluded transit-window duration [day]: {round_to_2(duration)}") + if fitsortext == 1: + display_aperture, display_annulus = reported_photometry_aperture_radii(photometry_info) + if bestCompStar == 'ensemble': + log_info(" Stellar Variability Reference Star: ensemble") + elif bestCompStar is not None: + log_info(f" Stellar Variability Reference Star: #{bestCompStar} - {comp_coords}") + else: + log_info(" Stellar Variability Reference Star: None") + if photometry_info.get('min_aperture') == 0: + log_info(" Optimal Method: PSF photometry") + else: + log_info(f" Optimal Aperture: {abs(np.round(display_aperture, 2))}") + log_info(f" Optimal Annulus: {np.round(display_annulus, 2)}") + log_info("*********************************************************") + + if vsp_params: + AIDoutput_files = AIDOutputFiles(myfit, pDict, exotic_infoDict, auid, chart_id, vsp_params) + output_files = OutputFiles(myfit, pDict, exotic_infoDict, durs) + error_txt = "\n\tPlease report this issue on the Exoplanet Watch Slack Channel in #data-reductions." + + try: + phase = np.asarray(getattr(myfit, 'phase', get_phase(myfit.time, pDict['pPer'], pDict['midT']))) + output_files.differential_magnitude() + output_files.stellar_variability_differential_magnitude() + output_files.final_lightcurve(phase) + except Exception as e: + log_info(f"\nError: Could not create FinalLightCurve.csv. {error_txt}\n\t{e}", error=True) + try: + if fitsortext == 1: + display_aperture, display_annulus = reported_photometry_aperture_radii(photometry_info) + output_files.final_planetary_params(phot_opt=True, vsp_params=vsp_params, + comp_star=bestCompStar, comp_coords=comp_coords, + min_aper=np.round(display_aperture, 2), + min_annul=np.round(display_annulus, 2), + adaptive_summary=photometry_info.get('adaptive_summary'), + photometry_info=photometry_info, + publish_to_root=True) + else: + output_files.final_planetary_params( + phot_opt=False, + vsp_params=vsp_params, + publish_to_root=True, + ) + except Exception as e: + log_info(f"\nError: Could not create FinalParams.json. {error_txt}\n\t{e}", error=True) + try: + if fitsortext == 1: + output_files.plate_status(plateStatus) + except Exception as e: + log_info(f"\nError: Could not create plate_status.csv. {error_txt}\n\t{e}", error=True) + try: + if vsp_params: + AIDoutput_files.aavso() + except Exception as e: + log_info(f"\nError: Could not create AID_AAVSO.txt. {error_txt}\n\t{e}", error=True) + + log_info("Output Files Saved") + reduction_stage_timer.checkpoint("Output file generation") + + log_info("\n************************") + log_info("End of Reduction Process") + log_info("************************") + + log_info("\n\n************************") + log_info("EXOTIC has successfully run!!!") + log_info("It is now safe to close this window.") + log_info("************************") + + log.debug("Stopped ...") + return ########################## # NESTED SAMPLING FITTING @@ -2627,15 +36642,18 @@ def main(): } phase = (goodTimes - prior['tmid']) / prior['per'] - prior['tmid'] = pDict['midT'] + np.floor(phase).max() * prior['per'] - upper = pDict['midT'] + 35 * pDict['midTUnc'] + np.floor(phase).max() * (pDict['pPer'] + 35 * pDict['pPerUnc']) - lower = pDict['midT'] - 35 * pDict['midTUnc'] + np.floor(phase).max() * (pDict['pPer'] - 35 * pDict['pPerUnc']) - - # clip bounds so they're within 1 orbit - if upper > prior['tmid'] + 0.25*prior['per']: - upper = prior['tmid'] + 0.25*prior['per'] - if lower < prior['tmid'] - 0.25*prior['per']: - lower = prior['tmid'] - 0.25*prior['per'] + expected_duration = estimate_transit_duration_from_prior_geometry(prior) + ephemeris_tmid_search_summary = estimate_ephemeris_tmid_and_bounds( + goodTimes, + pDict['midT'], + prior['per'], + pDict['midTUnc'], + pDict['pPerUnc'], + expected_duration=expected_duration, + sigma_multiplier=35.0, + ) + prior['tmid'] = ephemeris_tmid_search_summary['tmid'] + lower, upper = ephemeris_tmid_search_summary['bounds'] if np.floor(phase).max() - np.floor(phase).min() == 0: log_info("Error: Estimated mid-transit not in observation range (check priors or observation time)", error=True) @@ -2643,26 +36661,159 @@ def main(): log_info(f" end:{np.max(goodTimes)}", error=True) log_info(f"prior:{prior['tmid']}", error=True) - mybounds = { - 'rprs': [0, prior['rprs'] * 1.25], - 'tmid': [lower, upper], - 'inc': [prior['inc'] - 5, min(90, prior['inc'] + 5)], - 'a2': [-3, 3], - } + if ephemeris_tmid_search_summary.get('duration_capped'): + log_info(ephemeris_tmid_search_summary['note']) + eebls_tmid_search_summary = None + if use_eebls_tmid_initializer: + eebls_tmid_search_summary = estimate_tmid_and_bounds_with_eebls( + goodTimes, + goodFluxes, + goodNormUnc, + prior, + [lower, upper], + ) + log_info(eebls_tmid_search_summary['note']) + if eebls_tmid_search_summary.get('applied'): + prior['tmid'] = eebls_tmid_search_summary['tmid'] + lower, upper = eebls_tmid_search_summary['bounds'] + + final_airmass_span = airmass_span(goodAirmasses) + airmass_skip_note = None + skip_final_airmass_fit = bool(exotic_infoDict.get('airmass_already_corrected')) + if skip_final_airmass_fit: + airmass_skip_note = ( + "Skipped (input AAVSO file already reports AIRMASS, AIRMASS CORRECTION FUNCTION); " + "no airmass correction applied." + ) + log_info( + "Input AAVSO file reports AIRMASS, AIRMASS CORRECTION FUNCTION; " + "skipping airmass fitting and applying no airmass correction." + ) + elif should_skip_airmass_fit(goodAirmasses): + skip_final_airmass_fit = True + log_info( + f"Airmass span {final_airmass_span:.4f} <= {AIRMASS_FLAT_RANGE_THRESHOLD:.2f}; " + "skipping airmass fitting and applying no airmass correction." + ) + + search_restriction_prior = enrich_search_restriction_prior_with_rprs_data_uncertainty( + build_search_restriction_prior_from_planet_dict(pDict), + goodTimes, + goodFluxes, + goodNormUnc, + prior, + context_label="final light curve", + ) + mybounds = build_initial_transit_bounds( + prior, + [lower, upper], + ars_unc=pDict.get('aRsUnc'), + rprs_data_uncertainty=search_restriction_prior.get('rprs_data_uncertainty'), + search_restriction_prior=search_restriction_prior, + ) + apply_vertical_flux_normalization_bound( + prior, + mybounds, + goodFluxes, + disable_vertical_flux_normalization, + ) + if not skip_final_airmass_fit: + mybounds['a2'] = [-3, 3] if np.isnan(goodFluxes).all(): log_info("Error: No valid photometry data found.", error=True) return - # final light curve fit - myfit = lc_fitter(goodTimes, goodFluxes, goodNormUnc, goodAirmasses, prior, mybounds, mode='ns') + if reuse_selected_final_model: + myfit = photometry_info['best_fit_lc'] + goodTimes = np.asarray(getattr(myfit, 'time', goodTimes), dtype=float) + goodAirmasses = np.asarray(getattr(myfit, 'airmass', goodAirmasses), dtype=float) + reused_flux = photometry_info.get('selected_fit_good_flux') + reused_unc = photometry_info.get('selected_fit_good_unc') + if reused_flux is not None and np.shape(reused_flux) == np.shape(goodTimes): + goodFluxes = np.asarray(reused_flux, dtype=float) + else: + goodFluxes = np.asarray(getattr(myfit, 'detrended', goodFluxes), dtype=float) + if reused_unc is not None and np.shape(reused_unc) == np.shape(goodTimes): + goodNormUnc = np.asarray(reused_unc, dtype=float) + else: + goodNormUnc = np.asarray(getattr(myfit, 'detrendederr', goodNormUnc), dtype=float) + log_info( + "Using the selected comparison-star full-reduction ultranest fit for final outputs; " + "no additional final nested-sampling fit is being run." + ) + else: + # final light curve fit + myfit, goodFluxes, goodNormUnc = fit_final_lightcurve_with_oot_baseline_detrending( + goodTimes, + goodFluxes, + goodNormUnc, + goodAirmasses, + prior, + mybounds, + exposure_times_seconds=goodExposureTimes, + skip_airmass_fit=skip_final_airmass_fit, + airmass_skip_note=airmass_skip_note, + disable_vertical_flux_normalization=disable_vertical_flux_normalization, + detrend_on_outoftransit_baseline=detrend_on_outoftransit_baseline, + use_impactparameter_rather_than_inclination_to_fit= + use_impactparameter_rather_than_inclination_to_fit, + plot_time_range=full_plot_time_range, + baseline_duration_multiplier=final_fit_baseline_duration_multiplier, + expected_planet_dict=pDict, + expected_tmid_search_summary=ephemeris_tmid_search_summary, + eebls_search_summary=eebls_tmid_search_summary, + search_restriction_prior=search_restriction_prior, + ) + if ( + reuse_selected_final_model + and getattr(myfit, 'sparse_posterior_live_point_extension_note', None) is None + ): + myfit = extend_selected_comparison_live_points_if_needed(myfit) + annotate_transit_detection_qc(myfit) # myfit.dataerr *= np.sqrt(myfit.chi2 / myfit.data.shape[0]) # scale errorbars by sqrt(rchi2) # myfit.detrendederr *= np.sqrt(myfit.chi2 / myfit.data.shape[0]) + if fitsortext != 1 and not vsp_params: + try: + calibration_label, calibration_star = nextastro_prereduced_calibration_star( + exotic_infoDict.get('phot_comp_star'), + exotic_infoDict.get('filter'), + ) + if calibration_star: + vsp_params = build_stellar_variability_params_from_fit( + myfit, + calibration_star, + calibration_star.get('pos'), + calibration_label, + exotic_infoDict['save'], + pDict['sName'], + observed_filter=exotic_infoDict.get('observed_filter', exotic_infoDict.get('filter')), + observation_date=exotic_infoDict.get('date'), + ) + if not auid: + auid = vsx_auid(pDict['ra'], pDict['dec']) + else: + log_info( + "\nWarning: Could not create pre-reduced stellar variability output because " + "no comparison-star RA/Dec with a usable NextAstro catalog magnitude was available.", + warn=True, + ) + except Exception as exc: + log_info( + f"\nWarning: Could not create pre-reduced stellar variability output " + f"({describe_retry_exception(exc)}).", + warn=True, + ) + # estimate transit duration pars = dict(**myfit.parameters) times = np.linspace(np.min(myfit.time), np.max(myfit.time), 1000) - data_highres = transit(times, pars) + fit_transit_model = getattr(myfit, '_transit_model', None) + if callable(fit_transit_model): + data_highres = fit_transit_model(times, pars) + else: + data_highres = transit(times, pars) dt = np.diff(times).mean() durs = [] for r in range(1000): @@ -2674,45 +36825,265 @@ def main(): tmask = data < 1 durs.append(tmask.sum() * dt) - plot_final_lightcurve(myfit, data_highres, pDict['pName'], exotic_infoDict['save'], exotic_infoDict['date']) + if vsp_params: + try: + myfit.stellar_variability_params = vsp_params + myfit.stellar_variability_target_name = pDict.get('sName', pDict.get('pName')) + myfit.stellar_variability_reference_label = vsp_params[0].get('cname') + except Exception: + pass + + plot_final_lightcurve( + myfit, + data_highres, + pDict['pName'], + exotic_infoDict['save'], + exotic_infoDict['date'], + observed_filter=exotic_infoDict.get('observed_filter', exotic_infoDict.get('filter')), + ) + diagnostics_dir = Path(exotic_infoDict['save']) / "Diagnostics" + plot_prior_posterior_comparison(myfit, pDict, pDict['pName'], diagnostics_dir, exotic_infoDict['date']) + plot_ktmf_qc_metrics(myfit, pDict['pName'], diagnostics_dir, exotic_infoDict['date']) if fitsortext == 1: - plot_obs_stats(myfit, exotic_infoDict['comp_stars'], psf_data, si, gi, pDict['pName'], - exotic_infoDict['save'], exotic_infoDict['date']) + observing_background_series = build_observing_background_series( + psf_data, + aper_data, + photometry_info, + len(science_comp_stars), + ) + plot_obs_stats(myfit, science_comp_stars, psf_data, obs_stats_sort_index, + obs_stats_keep_mask, pDict['pName'], + exotic_infoDict['save'], exotic_infoDict['date'], + relative_flux_mask=None, + background_series=observing_background_series) ####################################################################### # print final extracted planetary parameters ####################################################################### + transit_qc = getattr(myfit, 'transit_qc', None) + qc_status = None + qc_summary = None + qc_ktmf_metric = np.nan + if isinstance(transit_qc, dict) and transit_qc: + qc_status = str(transit_qc.get('status', 'unknown')).upper() + qc_summary = transit_qc.get('summary') + qc_ktmf_metric = _finite_float(transit_qc.get('ktmf_metric'), default=np.nan) + log_info("\n*********************************************************") log_info("FINAL PLANETARY PARAMETERS\n") - log_info(f" Mid-Transit Time [BJD_TDB]: {round_to_2(myfit.parameters['tmid'], myfit.errors['tmid'])} +/- {round_to_2(myfit.errors['tmid'])}") - log_info(f" Radius Ratio (Planet/Star) [Rp/R*]: {round_to_2(myfit.parameters['rprs'], myfit.errors['rprs'])} +/- {round_to_2(myfit.errors['rprs'])}") - log_info(f" Transit depth [(Rp/R*)^2]: {round_to_2(100. * (myfit.parameters['rprs'] ** 2.))} +/- {round_to_2(100. * 2. * myfit.parameters['rprs'] * myfit.errors['rprs'])} [%]") - log_info(f" Orbital Inclination [inc]: {round_to_2(myfit.parameters['inc'], myfit.errors['inc'])} +/- {round_to_2(myfit.errors['inc'])}") - log_info(f" Airmass coefficient 1: {round_to_2(myfit.parameters['a1'], myfit.errors['a1'])} +/- {round_to_2(myfit.errors['a1'])}") - log_info(f" Airmass coefficient 2: {round_to_2(myfit.parameters['a2'], myfit.errors['a2'])} +/- {round_to_2(myfit.errors['a2'])}") - log_info(f" Residual scatter: {round_to_2(100. * np.std(myfit.residuals / np.median(myfit.data)))} %") + if qc_status: + if qc_summary: + log_info(f" Transit detection QC: {qc_status} - {qc_summary}") + else: + log_info(f" Transit detection QC: {qc_status}") + if np.isfinite(qc_ktmf_metric): + log_info(f" KTMF: {qc_ktmf_metric:.2f} / 5.00") + empirical_uncertainty = getattr(myfit, 'empirical_transit_uncertainty', None) + if not isinstance(empirical_uncertainty, dict) or not empirical_uncertainty.get('available'): + empirical_uncertainty = fit_empirical_transit_uncertainty(myfit) + rprs_model_error = _finite_float(getattr(myfit, 'errors', {}).get('rprs', np.nan), default=np.nan) + rprs_prior_error = _finite_float(pDict.get('rprsUnc', np.nan), default=np.nan) + rprs_error_fallback = rprs_model_error if np.isfinite(rprs_model_error) else rprs_prior_error + rprs_report_error = _finite_float( + (empirical_uncertainty or {}).get('combined_rprs_uncertainty'), + default=rprs_error_fallback, + ) + if not np.isfinite(rprs_report_error) or rprs_report_error < 0: + rprs_report_error = rprs_error_fallback + tmid_report_error = fit_parameter_model_data_uncertainty( + myfit, + 'tmid', + empirical_uncertainty=empirical_uncertainty, + ) + if not np.isfinite(tmid_report_error) or tmid_report_error < 0: + tmid_report_error = myfit.errors['tmid'] + inc_report_error = fit_parameter_model_data_uncertainty( + myfit, + 'inc', + empirical_uncertainty=empirical_uncertainty, + ) + if not np.isfinite(inc_report_error) or inc_report_error < 0: + inc_report_error = myfit.errors['inc'] + ars_report_error = fit_parameter_model_data_uncertainty( + myfit, + 'ars', + empirical_uncertainty=empirical_uncertainty, + ) + if not np.isfinite(ars_report_error) or ars_report_error < 0: + ars_report_error = myfit.errors.get('ars', np.nan) + log_info( + " Mid-Transit Time [BJD_TDB]: " + f"{format_value_with_uncertainty(myfit.parameters['tmid'], tmid_report_error)}" + ) + log_info( + " Radius Ratio (Planet/Star) [Rp/R*]: " + f"{format_value_with_uncertainty(myfit.parameters['rprs'], rprs_report_error)}" + ) + rprs_prior_fallback_note = getattr(myfit, 'rprs_prior_fallback_note', None) + if rprs_prior_fallback_note: + log_info(f" Rp/R* fallback note: {rprs_prior_fallback_note}") + for depth_label, depth_text in formatted_transit_depth_parameters( + myfit, + pDict, + empirical_uncertainty=empirical_uncertainty, + ).items(): + log_info(f" {depth_label}: {depth_text}") + log_info( + " Orbital Inclination [inc]: " + f"{format_value_with_uncertainty(myfit.parameters['inc'], inc_report_error)}" + ) + ars_text = format_parameter_with_error(myfit.parameters.get('ars'), ars_report_error) + if ars_text is not None: + log_info(f" Ratio of Distance to Stellar Radius [a/Rs]: {ars_text}") + impact_error_overrides = { + 'ars': ars_report_error, + 'inc': inc_report_error, + } + model_impact_parameter, model_impact_error = fit_impact_parameter_value_error(myfit) + if ( + np.isfinite(model_impact_error) + and ('b' in (getattr(myfit, 'parameters', {}) or {}) + or 'b' in (getattr(myfit, 'sample_parameters', {}) or {})) + ): + impact_error_overrides['b'] = model_impact_error * empirical_red_noise_error_scale( + empirical_uncertainty + ) + impact_parameter, impact_error = fit_impact_parameter_value_error( + myfit, + errors_override=impact_error_overrides, + ) + impact_text = format_parameter_with_error(impact_parameter, impact_error) + if impact_text is not None: + log_info(f" Impact Parameter [b]: {impact_text}") + if getattr(myfit, 'airmass_fit_skipped', False): + log_info(f" Airmass correction: {myfit.airmass_correction_note}") + else: + fit_parameters = getattr(myfit, 'parameters', {}) or {} + fit_errors = getattr(myfit, 'errors', {}) or {} + airmass_scale_key = 'a1' if 'a1' in fit_parameters else 'a0' + fixed_after_detrending = baseline_fixed_after_detrending(myfit) + pre_detrending_report = pre_detrending_baseline_report(myfit) + if fixed_after_detrending and pre_detrending_report is not None: + if pre_detrending_report.get('source'): + log_info( + " Pre-detrending baseline source: " + f"{pre_detrending_report['source']}" + ) + if pre_detrending_report.get('scale_text'): + scale_label = ( + "baseline flux (a0)" + if pre_detrending_report['scale_parameter'] == 'a0' + else "airmass coefficient 1 (a1)" + ) + log_info( + f" Pre-detrending {scale_label}: " + f"{pre_detrending_report['scale_text']}" + ) + if pre_detrending_report.get('a2_text'): + log_info( + " Pre-detrending airmass coefficient 2 (a2): " + f"{pre_detrending_report['a2_text']}" + ) + log_info(" Final detrended-fit baseline coefficients:") + if airmass_scale_key in fit_parameters: + airmass_scale_error = fit_errors.get(airmass_scale_key) + if fixed_after_detrending: + log_info( + f" Airmass coefficient 1: " + f"{round_to_2(fit_parameters[airmass_scale_key])} " + "(fixed after out-of-transit baseline detrending)" + ) + elif airmass_scale_error is not None and np.isfinite(airmass_scale_error): + log_info( + f" Airmass coefficient 1: " + f"{format_value_with_uncertainty(fit_parameters[airmass_scale_key], airmass_scale_error)}" + ) + else: + log_info( + f" Airmass coefficient 1: " + f"{round_to_2(fit_parameters[airmass_scale_key])} (fixed; uncertainty unavailable)" + ) + if 'a2' in fit_parameters: + a2_error = fit_errors.get('a2') + if fixed_after_detrending: + log_info( + f" Airmass coefficient 2: " + f"{round_to_2(fit_parameters['a2'])} " + "(fixed after out-of-transit baseline detrending)" + ) + elif a2_error is not None and np.isfinite(a2_error): + log_info( + f" Airmass coefficient 2: " + f"{format_value_with_uncertainty(fit_parameters['a2'], a2_error)}" + ) + else: + log_info( + f" Airmass coefficient 2: " + f"{round_to_2(fit_parameters['a2'])} (fixed; uncertainty unavailable)" + ) + if isinstance(transit_qc, dict) and transit_qc: + residual_scatter = transit_qc.get('residual_scatter', np.nan) + if np.isfinite(residual_scatter): + log_info(f"Residual scatter around full model fit: {residual_scatter * 100.0:.4f}%") + if np.isfinite(transit_qc.get('deviation_from_expected_value', np.nan)): + log_info( + f" Deviation From Expected Value: {transit_qc['deviation_from_expected_value']:.2f} / 1.00" + ) + if np.isfinite(transit_qc.get('rprs_deviation_sigma', np.nan)): + log_info( + f" Expected-value Rp/R* sigma: {transit_qc['rprs_deviation_sigma']:.2f}" + ) + for contribution in transit_qc.get('ktmf_contributions', []): + log_info(f" {format_ktmf_contribution(contribution)}") if fitsortext == 1: + if np.isfinite(photometry_info.get('calibration_field_score', np.inf)): + log_info(f" Comparison-Star Field Score: {round_to_2(100. * photometry_info['calibration_field_score'])} %") + display_aperture, display_annulus = reported_photometry_aperture_radii(photometry_info) + adaptive_summary = photometry_info.get('adaptive_summary') if photometry_info['min_aperture'] >= 0: - log_info(f" Best Comparison Star: #{bestCompStar} - {comp_coords}") + if bestCompStar == 'ensemble': + log_info(" Transit Fit Comparison Star: ensemble") + else: + log_info(f" Transit Fit Comparison Star: #{bestCompStar} - {comp_coords}") else: - log_info(" Best Comparison Star: None") + log_info(" Transit Fit Comparison Star: None") if photometry_info['min_aperture'] == 0: log_info(" Optimal Method: PSF photometry") else: - log_info(f" Optimal Aperture: {abs(np.round(photometry_info['min_aperture'], 2))}") - log_info(f" Optimal Annulus: {np.round(photometry_info['min_annulus'], 2)}") - log_info(f" Transit Duration [day]: {round_to_2(np.mean(durs), np.std(durs))} +/- {round_to_2(np.std(durs))}") + if adaptive_summary is not None: + log_info(f" Optimal Aperture: {abs(display_aperture):.2f} +/- {adaptive_summary['aperture_std']:.2f} px") + log_info(f" Optimal Annulus: {display_annulus:.2f} +/- {adaptive_summary['annulus_std']:.2f} px") + log_info(f" Adaptive Aperture Scale: {adaptive_summary['aperture_sigma']:.2f} sigma") + log_info(f" Adaptive Annulus Scale: {adaptive_summary['annulus_sigma']:.2f} sigma") + log_info(f" Aperture Range: {adaptive_summary['aperture_min']:.2f} to {adaptive_summary['aperture_max']:.2f} px") + log_info(f" Annulus Range: {adaptive_summary['annulus_min']:.2f} to {adaptive_summary['annulus_max']:.2f} px") + else: + log_info(f" Optimal Aperture: {abs(np.round(display_aperture, 2))}") + log_info(f" Optimal Annulus: {np.round(display_annulus, 2)}") + log_info( + " Transit Duration [day]: " + f"{format_value_with_uncertainty(np.mean(durs), np.std(durs))}" + ) log_info("*********************************************************") ########## # SAVE DATA ########## - fig = myfit.plot_triangle() - fig.savefig(Path(exotic_infoDict['save']) / "temp" / - f"Triangle_{pDict['pName']}_{exotic_infoDict['date']}.png") + selected_triangle_source_dir = ( + photometry_info.get('selected_fit_final_output_dir') + if reuse_selected_final_model + else None + ) + save_final_triangle_plot( + myfit, + exotic_infoDict['save'], + pDict['pName'], + exotic_infoDict['date'], + source_dir=selected_triangle_source_dir, + ) if vsp_params: AIDoutput_files = AIDOutputFiles(myfit, pDict, exotic_infoDict, auid, chart_id, vsp_params) @@ -2721,23 +37092,95 @@ def main(): try: phase = get_phase(myfit.time, pDict['pPer'], myfit.parameters['tmid']) + output_files.differential_magnitude() + output_files.stellar_variability_differential_magnitude() output_files.final_lightcurve(phase) except Exception as e: log_info(f"\nError: Could not create FinalLightCurve.csv. {error_txt}\n\t{e}", error=True) try: if fitsortext == 1: + display_aperture, display_annulus = reported_photometry_aperture_radii(photometry_info) output_files.final_planetary_params(phot_opt=True, vsp_params=vsp_params, comp_star=bestCompStar, comp_coords=comp_coords, - min_aper=np.round(photometry_info['min_aperture'], 2), - min_annul=np.round(photometry_info['min_annulus'], 2)) + min_aper=np.round(display_aperture, 2), + min_annul=np.round(display_annulus, 2), + adaptive_summary=photometry_info.get('adaptive_summary'), + photometry_info=photometry_info, + publish_to_root=True) else: - output_files.final_planetary_params(phot_opt=False, vsp_params=vsp_params) + output_files.final_planetary_params( + phot_opt=False, + vsp_params=vsp_params, + publish_to_root=True, + ) except Exception as e: log_info(f"\nError: Could not create FinalParams.json. {error_txt}\n\t{e}", error=True) try: - if bestCompStar: + if isinstance(bestCompStar, int): exotic_infoDict['phot_comp_star'] = save_comp_ra_dec(wcs_file, ra_wcs, dec_wcs, comp_coords) - output_files.aavso(exotic_infoDict['phot_comp_star'], goodAirmasses, ld0, ld1, ld2, ld3, epw_md5) + aavso_photometry_info = photometry_info if fitsortext == 1 else None + aavso_frame_filtering_info = None + aavso_astrometry_info = None + aavso_bad_pixel_info = None + if fitsortext == 1: + aavso_frame_filtering_info = { + 'initial_frame_count': precheck_inputfile_count, + 'after_missing_wcs_filter_frame_count': post_wcs_inputfile_count, + 'after_target_wcs_filter_frame_count': post_target_wcs_inputfile_count, + 'final_prephotometry_frame_count': post_pointing_inputfile_count, + 'ignore_header_wcs': ignore_header_wcs, + 'bad_wcs_threshold_percent': ( + 100.0 * bad_wcs_threshold_fraction + if np.isfinite(bad_wcs_threshold_fraction) + else np.nan + ), + 'pointing_rejection_sigma': pointing_rejection_sigma, + 'dropped_missing_wcs_files': dropped_wcs_files, + 'dropped_target_wcs_files': dropped_target_wcs_files, + 'dropped_pointing_files': dropped_pointing_files, + } + aavso_astrometry_info = { + 'wcs_file': str(wcs_file) if wcs_file else None, + 'coordinate_source': 'wcs' if wcs_file else 'input_pixels', + 'ignore_header_wcs': ignore_header_wcs, + 'plate_solution_option': exotic_infoDict.get('plate_opt'), + 'target_input_pixel': exotic_infoDict.get('tar_coords'), + 'comparison_input_pixels': exotic_infoDict.get('comp_stars'), + 'comparison_input_ra_dec_deg': provided_comparison_radec, + 'target_ra_dec_deg': ra_dec_tar, + 'comparison_ra_dec_deg': ra_dec_wcs, + 'catalog_ra_dec_deg': [pDict.get('ra'), pDict.get('dec')], + 'gaia_distance_pc': pDict.get('dist'), + 'proper_motion_ra_mas_yr': pDict.get('pm_ra'), + 'proper_motion_dec_mas_yr': pDict.get('pm_dec'), + } + aavso_bad_pixel_info = { + 'enabled': bool(detect_bad_pixels_before_photometry), + 'detected': bad_pixel_reference is not None, + } + if bad_pixel_reference is not None: + bad_pixel_mask = np.asarray(bad_pixel_reference.get('mask'), dtype=bool) + aavso_bad_pixel_info.update({ + 'bad_pixel_count': int(np.count_nonzero(bad_pixel_mask)), + 'frame_count': bad_pixel_reference.get('frame_count'), + 'required_count': bad_pixel_reference.get('required_count'), + 'minimum_fraction': bad_pixel_reference.get('minimum_fraction'), + 'counts_path': bad_pixel_reference.get('counts_path'), + 'mask_path': bad_pixel_reference.get('mask_path'), + }) + output_files.aavso( + exotic_infoDict['phot_comp_star'], + goodAirmasses, + ld0, + ld1, + ld2, + ld3, + epw_md5, + photometry_info=aavso_photometry_info, + astrometry_info=aavso_astrometry_info, + frame_filtering_info=aavso_frame_filtering_info, + bad_pixel_info=aavso_bad_pixel_info, + ) except Exception as e: log_info(f"\nError: Could not create AAVSO.txt. {error_txt}\n\t{e}", error=True) try: @@ -2751,6 +37194,7 @@ def main(): log_info(f"\nError: Could not create plate_status.csv. {error_txt}\n\t{e}", error=True) log_info("Output Files Saved") + reduction_stage_timer.checkpoint("Final transit analysis and output file generation") log_info("\n************************") log_info("End of Reduction Process") @@ -2764,18 +37208,50 @@ def main(): log.debug("Stopped ...") +def main(): + global _UNHANDLED_EXCEPTION_LOGGED + + _UNHANDLED_EXCEPTION_LOGGED = False + previous_logging_raise_exceptions = logging.raiseExceptions + # Python's logging package prints its own ``--- Logging error ---`` + # traceback when any handler fails and this development flag is true. + # Notebook transports and mounted Drive files can disconnect independently + # of the reduction, so suppress all such internal logging tracebacks for the + # duration of the run. Genuine EXOTIC exceptions are still reported by the + # explicit exception handling below. + logging.raiseExceptions = False + try: + configure_runtime_logging(start_new_run=True) + install_exception_hooks() + + try: + return _main_impl() + except (KeyboardInterrupt, SystemExit): + raise + except Exception as exc: + _handle_unhandled_exception(type(exc), exc, exc.__traceback__) + raise + finally: + cancel_runtime_traceback_watchdog() + close_runtime_logging() + finally: + logging.raiseExceptions = previous_logging_raise_exceptions + + +def cli(): + global _UNHANDLED_EXCEPTION_LOGGED + + _UNHANDLED_EXCEPTION_LOGGED = False + install_exception_hooks() + + try: + return main() + except (KeyboardInterrupt, SystemExit): + raise + except Exception as exc: + _handle_unhandled_exception(type(exc), exc, exc.__traceback__) + raise + + if __name__ == "__main__": - # configure logger for standalone execution - logging.root.setLevel(logging.DEBUG) - fileFormatter = logging.Formatter("%(asctime)s.%(msecs)03d [%(threadName)-12.12s] %(levelname)-5.5s " - "%(funcName)s:%(lineno)d - %(message)s", f"%Y-%m-%dT%H:%M:%S") - fileHandler = TimedRotatingFileHandler(filename="exotic.log", when="midnight", backupCount=2) - fileHandler.setLevel(logging.DEBUG) - fileHandler.setFormatter(fileFormatter) - consoleFormatter = logging.Formatter("%(message)s") - consoleHandler = logging.StreamHandler(sys.stdout) - consoleHandler.setFormatter(consoleFormatter) - consoleHandler.setLevel(logging.INFO) - log.addHandler(fileHandler) - log.addHandler(consoleHandler) - main() + raise SystemExit(cli()) diff --git a/exotic/exotic_gui.py b/exotic/exotic_gui.py index 54172eda..ad8293db 100644 --- a/exotic/exotic_gui.py +++ b/exotic/exotic_gui.py @@ -48,6 +48,7 @@ import os import platform import python_version +import re import subprocess import sys @@ -77,6 +78,11 @@ except ImportError: # package import from version import __version__ +try: + from .inputs import parse_aavso_comp_star, parse_aavso_prereduced_overrides +except ImportError: + from inputs import parse_aavso_comp_star, parse_aavso_prereduced_overrides + animate_toggle() @@ -122,6 +128,49 @@ def file_path(self): return self.filePath.get() +def stringify_prefill(value): + if value is None: + return "" + return str(value) + + +def normalize_filter_option_lookup(value): + if value is None: + return None + return re.sub(r'[\W_]+', '', str(value).strip().lower()) + + +def preselected_filter_option(prefill, choices): + if not isinstance(prefill, dict): + return choices[0] + + filter_desc = prefill.get('filter_desc') + filter_value = prefill.get('filter') + if filter_desc in photometric_filters: + return filter_desc + if filter_value in photometric_filters: + return filter_value + + normalized_candidates = { + normalize_filter_option_lookup(filter_desc), + normalize_filter_option_lookup(filter_value), + } + normalized_candidates.discard(None) + + for option in choices: + if option == "N/A": + continue + option_metadata = photometric_filters.get(option, {}) + if normalize_filter_option_lookup(option) in normalized_candidates: + return option + if normalize_filter_option_lookup(option_metadata.get('name')) in normalized_candidates: + return option + + if prefill.get('wl_min') and prefill.get('wl_max'): + return "N/A" + return choices[0] + + def main(): try: python_version.check(min=(3, 8, 0), max=(4, 0, 0)) @@ -258,7 +307,10 @@ def main(): # "Planet Name": "HAT-P-32 b", planet_label = tk.Label(root, text="Planet Name", justify=tk.LEFT) planet_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - planet_entry.insert(tk.END, "HAT-P-32 b") + planet_entry.insert( + tk.END, + stringify_prefill(input_data.get('aavso_prefill', {}).get('planet')) or "HAT-P-32 b", + ) planet_label.grid(row=i, column=j, sticky=tk.W, pady=2) planet_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -352,7 +404,7 @@ def save_input(): "Comment4": "and is only here to serve as a guide. Will be updated per user's advice.", "Image Calibrations Directory Guide": "Enter in the path to image calibrations or enter in null for none.", "Planetary Parameters Guide": "For planetary parameters that are not filled in, enter in null.", - "Comparison Star(s) Guide": "Up to 10 comparison stars can be added following the format given below.", + "Comparison Star(s) Guide": "Provide comparison stars either as X/Y pixels or as RA/Dec coordinates, but not both. RA/Dec requires a usable reference-image WCS.", "Obs. Latitude Guide": "Indicate the sign (+ North, - South) before the degrees. Needs to be in decimal or HH:MM:SS format.", "Obs. Longitude Guide": "Indicate the sign (+ East, - West) before the degrees. Needs to be in decimal or HH:MM:SS format.", "Plate Solution": "For your image to be given a plate solution, type y.", @@ -363,6 +415,36 @@ def save_input(): "Target Star DEC": "Must be in +/-DD:MM:SS sexagesimal format with correct sign at the beginning (+ or -).", "Demosaic Format": "Optional control for handling Bayer pattern color images - to use, provide Bayer color patttern of your camera (RGGB, BGGR, GRBG, GBRG) - null (no color processing) is default", "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", + "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", + "Pixel Alignment Fallback": "Set optional_info 'allow_pixel_alignment_fallback' to false to require WCS-only processing and drop every frame without celestial WCS. Default true; EXOTIC prefers WCS when coverage is consistent and otherwise retains the sequence for legacy alignment.", + "Bad WCS Threshold Percent": "When allow_pixel_alignment_fallback is true, set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS while still using WCS-only processing. Below the threshold, missing-WCS frames are dropped; at or above it, all frames are retained for alignment fallback. Default 3.", + "Prefer Pixel Coordinates Over WCS": "Set optional_info 'prefer_pixel_values_over_wcs_for_target' to y to keep the entered target pixel coordinates when they conflict with WCS-derived target coordinates. Default n.", + "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", + "Stellar Variability Only": "Set optional_info 'stellar_variability_only' to true to skip transit fitting, select comparison-star photometry by out-of-transit scatter, and discard predicted ingress-to-egress transit-window points. Default false.", + "Apparent Magnitudes Required": "Set optional_info 'require_apparent_magnitudes' to false when catalogue-calibrated apparent magnitudes are not required. Differential-magnitude products remain independent of catalogue calibration. Stellar-variability magnitudes use the raw target/reference ratio with no airmass correction. Default true.", + "Use Exactly Supplied Comparisons": "Set optional_info 'use_exactly_the_comps_provided' to true to use only the supplied X/Y or RA/Dec comparison coordinates with no replacement, addition, vetting, ranking, or ensemble-size limit. One comparison is used alone; multiple comparisons are all used as a fixed ensemble. Default false.", + "Maximum Transit Ensemble Comparisons": "Set optional_info 'maximum_number_of_ensemble_comparisons_for_transit' to the largest number of comparison stars used by the transit-fit ensemble. Default 5; minimum 2; no configured upper limit.", + "Maximum Stellar-Variability Ensemble Comparisons": "Set optional_info 'maximum_number_of_ensemble_comparisons_for_stellar_variability' to the largest number of comparison stars used by stellar-variability-only and fortuitous-variable ensembles. Default 5; minimum 2; no configured upper limit.", + "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into working_artifacts/, and median-8 repair those pixels before centroiding and photometry. Default n.", + "Multiprocess Bad-Pixel Precheck": "Set optional_info 'multiprocess_bad_pixel_precheck' to y or a positive process count to scan bad pixels in parallel. Default n.", + "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", + "Final Fit Baseline Duration Multiplier": "Set optional_info 'final_fit_baseline_duration_multiplier' to the number of fitted transit durations to keep as baseline before ingress and after egress during the automatic final-fit prefit/refit. Default 1.0.", + "EEBLS Tmid Initializer": "Set optional_info 'use_eebls_to_initialize_tmid_and_bounds' to y to run a fixed-period box least squares search over the light curve, use the strongest bracketed transit-like signal to initialize Tmid, and narrow the Tmid search range before fitting. Default y.", + "Pick Comparison by EEBLS SNR": "Set optional_info 'pick_comparison_by_eebls_snr' to y to prefer the comparison star whose target light curve yields the highest finite EEBLS SNR, falling back to residual scatter if no usable EEBLS SNR is available. Default y.", + "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", + "Maximum Rp/Rs Search Bound": "Set optional_info 'rprs_search_bound_max' to cap the nested-fit Rp/Rs search range. Default 0.5.", + "Restrict Rp/Rs Search Range": "Set optional_info 'restrict_Rp/Rs_range' to y to restrict Rp/Rs to a prior-centered percentage window. Set 'restrict_Rp/Rs_range_percentage' to control the half-width. Defaults y and 10.", + "Prior Rp/Rs Fallback For Pinned Posterior": "Set optional_info 'use_prior_Rp/Rs_when_posterior_pinned' to y to rerun a fit with Rp/Rs fixed to the input prior and quote a data-only Rp/Rs uncertainty when the Rp/Rs posterior remains edge-pinned after retry handling. Default y.", + "Restrict a/Rs Search Range": "Set optional_info 'restrict_a/Rs_range' to y to restrict a/Rs to a prior-centered percentage window. Set 'restrict_a/Rs_range_percentage' to control the half-width. Defaults y and 10.", + "Sparse Posterior Live-Point Retry": "Set optional_info 'use_sparse_posterior_live_point_retry' to y to rank comparison-star candidates at the configured UltraNest live-point count, then continue the chosen final comparison-star fit with 5x additional minimum live points using its retained final-pass bounds. Standalone final fits still only continue when Rp/Rs, Tmid, or a/Rs posteriors are too sparse. Set to n to disable. Default y.", + "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", + "Reject Overexposed Stars": "Set optional_info 'reject_overexposed_stars' to true to reject overexposed target frames and overexposed comparison-star measurements. Default true.", + "Saturation Value": "Set optional_info 'saturation_value' to the detector saturation value in the same units as the image pixels. If omitted/default, EXOTIC uses FITS SATURATE when available, maps TELESCOP Cecilia to 4096, otherwise uses 65535.", + "Overexposure Threshold Fraction": "Set optional_info 'overexposure_threshold_fraction' to the fraction of saturation used for rejection. Default 0.9.", + "Photometry Noise Budget": "Optional noise terms for raw-image photometry: gain_electrons_per_adu, read_noise_electrons, dark_current_electrons_per_second_per_pixel, flat_field_fractional_error, telescope_aperture_m, and scintillation_coefficient. Leave null to ignore an optional term.", + "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", + "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", + "Boolean Values": "All boolean settings accept JSON true/false, numeric 1/0, or case-insensitive strings y/n. The equivalent strings yes/no and on/off are also accepted.", "Formatting of null": "Due to the file being a .json, null is case sensitive and must be spelled as shown.", "Decimal Format": "Leading zero must be included when appropriate (Ex: 0.32, .32 or 00.32 causes errors.)." } @@ -371,12 +453,45 @@ def save_input(): "Target Star X & Y Pixel": input_data['targetpos'], "Comparison Star(s) X & Y Pixel": [input_data['comppos']], + "Comparison Star(s) RA & Dec": null, "Demosaic Format": null, # TODO add GUI input for these "Demosaic Output": null } new_inits['planetary_parameters'] = { "Planet Name": input_data['pName'], } + new_inits['optional_info'] = { + "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", + "allow_pixel_alignment_fallback": True, + "bad_wcs_threshold_percent": 3.0, + "prefer_pixel_values_over_wcs_for_target": "n", + "disable vertical flux normalization": False, + "stellar_variability_only": False, + "require_apparent_magnitudes": True, + "use_exactly_the_comps_provided": False, + "maximum_number_of_ensemble_comparisons_for_transit": 5, + "maximum_number_of_ensemble_comparisons_for_stellar_variability": 5, + "detect_bad_pixels_before_photometry": "n", + "multiprocess_bad_pixel_precheck": "n", + "detrend_on_outoftransit_baseline": True, + "final_fit_baseline_duration_multiplier": 1.0, + "use_eebls_to_initialize_tmid_and_bounds": "y", + "pick_comparison_by_eebls_snr": "y", + "use_impactparameter_rather_than_inclination_to_fit": "y", + "rprs_search_bound_max": 0.5, + "restrict_Rp/Rs_range": "y", + "restrict_Rp/Rs_range_percentage": 10.0, + "use_prior_Rp/Rs_when_posterior_pinned": "y", + "restrict_a/Rs_range": "y", + "restrict_a/Rs_range_percentage": 10.0, + "use_sparse_posterior_live_point_retry": "y", + "use_adaptive_apertures": False, + "reject_overexposed_stars": True, + "saturation_value": 65535, + "overexposure_threshold_fraction": 0.9, + "Use target-driven comp selection rather than comp-driven comp selection": "n", + "require_comp_star": "y" + } now = datetime.now() dt_string = now.strftime("%d_%m_%Y__%H_%M_%S") @@ -498,41 +613,29 @@ def save_input(): exp_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 - comp_star_label = tk.Label(root, text="Comparison Star used in Photometry (leave blank if none):", - justify=tk.LEFT) - comp_star_label.grid(row=i, column=j, sticky=tk.W, pady=2) - i += 1 - - comp_star_ra_label = tk.Label(root, text="Comparison Star RA", justify=tk.LEFT) - comp_star_ra_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - comp_star_ra_label.grid(row=i, column=j, sticky=tk.W, pady=2) - comp_star_ra_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) - i += 1 - - comp_star_dec_label = tk.Label(root, text="Comparison Star DEC", justify=tk.LEFT) - comp_star_dec_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - comp_star_dec_label.grid(row=i, column=j, sticky=tk.W, pady=2) - comp_star_dec_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) - i += 1 - - comp_star_x_label = tk.Label(root, text="Comparison Star X Pixel Coordinate", justify=tk.LEFT) - comp_star_x_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - comp_star_x_label.grid(row=i, column=j, sticky=tk.W, pady=2) - comp_star_x_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) - i += 1 - - comp_star_y_label = tk.Label(root, text="Comparison Star Y Pixel Coordinate", justify=tk.LEFT) - comp_star_y_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - comp_star_y_label.grid(row=i, column=j, sticky=tk.W, pady=2) - comp_star_y_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) + comp_star_note = tk.Label( + root, + text="Leave these blank to load time, units, exposure, filter, and comparison-star metadata from an AAVSO header when available.", + justify=tk.LEFT + ) + comp_star_note.grid(row=i, column=j, columnspan=2, sticky=tk.W, pady=2) i += 1 def save_input(): - input_data['file_time'] = pretime_entry.get() - input_data['file_units'] = preunit_entry.get() - input_data['exp'] = float(exp_entry.get()) - input_data['phot_comp_star'] = {'ra': comp_star_ra_entry.get(), 'dec': comp_star_dec_entry.get(), - 'x': comp_star_x_entry.get(), 'y': comp_star_y_entry.get()} + aavso_prefill = parse_aavso_prereduced_overrides(prered_file.file_path) + exposure_text = exp_entry.get().strip() + + input_data['aavso_prefill'] = aavso_prefill + input_data['file_time'] = pretime_entry.get().strip() or stringify_prefill(aavso_prefill.get('file_time')) + input_data['file_units'] = preunit_entry.get().strip() or stringify_prefill(aavso_prefill.get('file_units')) + input_data['exp'] = float(exposure_text) if exposure_text else aavso_prefill.get('exposure') + input_data['phot_comp_star'] = parse_aavso_comp_star(prered_file.file_path) + input_data['filtermin'] = aavso_prefill.get('wl_min') + input_data['filtermax'] = aavso_prefill.get('wl_max') + input_data['obs_name'] = stringify_prefill(aavso_prefill.get('obs_name')) + input_data['dist'] = aavso_prefill.get('dist') + input_data['pm_ra'] = aavso_prefill.get('pm_ra') + input_data['pm_dec'] = aavso_prefill.get('pm_dec') root.destroy() # Button for closing @@ -592,6 +695,7 @@ def save_input(): # Set up rows + columns i = 1 j = 0 + aavso_prefill = input_data.get('aavso_prefill', {}) if fitsortext.get() == 2 else {} folderPath = tk.StringVar() @@ -667,7 +771,7 @@ def save_input(): # # "AAVSO Observer Code (blank if none)": "RTZ", obscode_label = tk.Label(root, text="AAVSO Observer Code (leave blank if none)", justify=tk.LEFT) obscode_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - obscode_entry.insert(tk.END, "") + obscode_entry.insert(tk.END, stringify_prefill(aavso_prefill.get('aavso_num'))) obscode_label.grid(row=i, column=j, sticky=tk.W, pady=2) obscode_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -676,7 +780,7 @@ def save_input(): # # "Secondary Observer Codes (blank if none)": "", secondobscode_label = tk.Label(root, text="Secondary Observer Codes (leave blank if none)", justify=tk.LEFT) secondobscode_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - secondobscode_entry.insert(tk.END, "") + secondobscode_entry.insert(tk.END, stringify_prefill(aavso_prefill.get('second_obs'))) secondobscode_label.grid(row=i, column=j, sticky=tk.W, pady=2) secondobscode_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -684,28 +788,43 @@ def save_input(): # # "Observation date": "17-December-2017", obsdate_label = tk.Label(root, text="Observation date (e.g. DAY-MONTH-YEAR)", justify=tk.LEFT) obsdate_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) + obsdate_entry.insert(tk.END, stringify_prefill(aavso_prefill.get('date'))) obsdate_label.grid(row=i, column=j, sticky=tk.W, pady=2) obsdate_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 # # # "Obs. Latitude": "+32.41638889", - lat_label = tk.Label(root, text="Obs. Latitude (+ = North; - = South; e.g. +32.41)", justify=tk.LEFT) + lat_label_text = "Obs. Latitude (+ = North; - = South; e.g. +32.41)" + if fitsortext.get() == 2: + lat_label_text += " [optional for pre-reduced runs]" + lat_label = tk.Label(root, text=lat_label_text, justify=tk.LEFT) lat_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) + lat_entry.insert(tk.END, stringify_prefill(aavso_prefill.get('lat'))) lat_label.grid(row=i, column=j, sticky=tk.W, pady=2) lat_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 # # # "Obs. Longitude": "-110.73444444", - long_label = tk.Label(root, text="Obs. Longitude (+ = East; - = West; e.g. -110.74) ", justify=tk.LEFT) + long_label_text = "Obs. Longitude (+ = East; - = West; e.g. -110.74)" + if fitsortext.get() == 2: + long_label_text += " [optional for pre-reduced runs]" + long_label = tk.Label(root, text=long_label_text, justify=tk.LEFT) long_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) + long_entry.insert(tk.END, stringify_prefill(aavso_prefill.get('long'))) long_label.grid(row=i, column=j, sticky=tk.W, pady=2) long_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 # # # "Obs. Elevation (meters)": 2616, - elevation_label = tk.Label(root, text="Obs. Elevation [meters]", justify=tk.LEFT) + elevation_label_text = "Obs. Elevation [meters]" + if fitsortext.get() == 2: + elevation_label_text += " [optional for pre-reduced runs]" + elevation_label = tk.Label(root, text=elevation_label_text, justify=tk.LEFT) elevation_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - elevation_entry.insert(tk.END, "0") + if fitsortext.get() == 1: + elevation_entry.insert(tk.END, "0") + elif aavso_prefill.get('elev') is not None: + elevation_entry.insert(tk.END, stringify_prefill(aavso_prefill.get('elev'))) elevation_label.grid(row=i, column=j, sticky=tk.W, pady=2) elevation_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -717,6 +836,7 @@ def save_input(): "then note your actual camera type under \"Observing Notes\" below)", justify=tk.LEFT) cameratype_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) + cameratype_entry.insert(tk.END, stringify_prefill(aavso_prefill.get('camera'))) cameratype_label.grid(row=i, column=j, sticky=tk.W, pady=2) cameratype_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -724,6 +844,7 @@ def save_input(): # # "Pixel Binning": "1x1", pixbin_label = tk.Label(root, text="Pixel Binning (e.g 1x1)", justify=tk.LEFT) pixbin_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) + pixbin_entry.insert(tk.END, stringify_prefill(aavso_prefill.get('pixel_bin'))) pixbin_label.grid(row=i, column=j, sticky=tk.W, pady=2) pixbin_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -747,7 +868,7 @@ def save_input(): choices = [item for item in photometric_filters.keys()] + ["N/A"] choices = sorted(set(choices)) # sort and list unique values filteroptions = tk.StringVar(root) - filteroptions.set(choices[0]) # default value + filteroptions.set(preselected_filter_option(aavso_prefill, choices)) l3 = tk.Label(root, text='Filter (use N/A for custom)', justify=tk.LEFT) l3.grid(row=i, column=j, sticky=tk.W, pady=2) @@ -768,6 +889,7 @@ def save_input(): # "Observing Notes": "Weather, seeing was nice.", obsnotes_label = tk.Label(root, text="Observing Notes", justify=tk.LEFT) obsnotes_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) + obsnotes_entry.insert(tk.END, stringify_prefill(aavso_prefill.get('notes'))) obsnotes_label.grid(row=i, column=j, sticky=tk.W, pady=2) obsnotes_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -827,19 +949,24 @@ def save_input(): # root.mainloop() def save_input(): - input_data['obsnotes'] = obsnotes_entry.get() + input_data['obsnotes'] = obsnotes_entry.get().strip() or stringify_prefill(aavso_prefill.get('notes')) if filteroptions.get() == "N/A": - input_data['obsfilter'] = "N/A" + input_data['obsfilter'] = stringify_prefill(aavso_prefill.get('filter')) or "N/A" else: input_data['obsfilter'] = photometric_filters[filteroptions.get()]["name"] - input_data['pixbin'] = pixbin_entry.get() - input_data['cameratype'] = cameratype_entry.get() - input_data['obscode'] = obscode_entry.get() - input_data['secondobscode'] = secondobscode_entry.get() - input_data['obsdate'] = obsdate_entry.get() - input_data['lat'] = lat_entry.get() - input_data['long'] = long_entry.get() - input_data['elevation'] = float(elevation_entry.get()) + input_data['pixbin'] = pixbin_entry.get().strip() or stringify_prefill(aavso_prefill.get('pixel_bin')) + input_data['cameratype'] = cameratype_entry.get().strip() or stringify_prefill(aavso_prefill.get('camera')) + input_data['obscode'] = obscode_entry.get().strip() or stringify_prefill(aavso_prefill.get('aavso_num')) + input_data['secondobscode'] = secondobscode_entry.get().strip() or stringify_prefill(aavso_prefill.get('second_obs')) + input_data['obsdate'] = obsdate_entry.get().strip() or stringify_prefill(aavso_prefill.get('date')) + input_data['lat'] = lat_entry.get().strip() or stringify_prefill(aavso_prefill.get('lat')) + input_data['long'] = long_entry.get().strip() or stringify_prefill(aavso_prefill.get('long')) + elevation_value = elevation_entry.get().strip() + if fitsortext.get() == 1 or elevation_value: + input_data['elevation'] = float(elevation_value) + else: + input_data['elevation'] = aavso_prefill.get('elev') + input_data['obs_name'] = stringify_prefill(aavso_prefill.get('obs_name')) input_data['pixscale'] = pixscale_entry.get() if fitsortext.get() == 1: input_data['comppos'] = str(list(ast.literal_eval(comppos_entry.get()))) @@ -867,7 +994,7 @@ def save_input(): pass try: - if filteroptions.get() == "N/A": + if filteroptions.get() == "N/A" and (input_data.get('filtermin') is None or input_data.get('filtermax') is None): root=tk.Tk() root.protocol("WM_DELETE_WINDOW", exit) root.title(f"EXOTIC v{__version__}") @@ -886,6 +1013,8 @@ def save_input(): # "Filter Minimum Wavelength (nm)": null, filtermin_label = tk.Label(root, text="Filter Minimum Wavelength (nm)", justify=tk.LEFT) filtermin_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) + if input_data.get('filtermin') is not None: + filtermin_entry.insert(tk.END, stringify_prefill(input_data.get('filtermin'))) filtermin_label.grid(row=i, column=j, sticky=tk.W, pady=2) filtermin_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -893,13 +1022,17 @@ def save_input(): # "Filter Maximum Wavelength (nm)": null filtermax_label = tk.Label(root, text="Filter Maximum Wavelength (nm)", justify=tk.LEFT) filtermax_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) + if input_data.get('filtermax') is not None: + filtermax_entry.insert(tk.END, stringify_prefill(input_data.get('filtermax'))) filtermax_label.grid(row=i, column=j, sticky=tk.W, pady=2) filtermax_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 def save_input(): - input_data['filtermax'] = float(filtermax_entry.get()) - input_data['filtermin'] = float(filtermin_entry.get()) + filtermax_value = filtermax_entry.get().strip() + filtermin_value = filtermin_entry.get().strip() + input_data['filtermax'] = float(filtermax_value) if filtermax_value else input_data.get('filtermax') + input_data['filtermin'] = float(filtermin_value) if filtermin_value else input_data.get('filtermin') root.destroy() # Button for closing @@ -970,7 +1103,10 @@ def save_input(): # "Planet Name": "HAT-P-32 b", planet_label = tk.Label(root, text="Planet Name", justify=tk.LEFT) planet_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - planet_entry.insert(tk.END, "HAT-P-32 b") + planet_entry.insert( + tk.END, + stringify_prefill(input_data.get('aavso_prefill', {}).get('planet')) or "HAT-P-32 b", + ) planet_label.grid(row=i, column=j, sticky=tk.W, pady=2) planet_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -978,7 +1114,10 @@ def save_input(): # "Host Star Name": "HAT-P-32", star_label = tk.Label(root, text="Host Star Name", justify=tk.LEFT) star_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - star_entry.insert(tk.END, "HAT-P-32") + star_entry.insert( + tk.END, + stringify_prefill(input_data.get('aavso_prefill', {}).get('host_star')) or "HAT-P-32", + ) star_label.grid(row=i, column=j, sticky=tk.W, pady=2) star_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -1227,7 +1366,10 @@ def save_input(): # "Planet Name": "HAT-P-32 b", planet_label = tk.Label(root, text="Planet Name", justify=tk.LEFT) planet_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - planet_entry.insert(tk.END, "HAT-P-32 b") + planet_entry.insert( + tk.END, + stringify_prefill(input_data.get('aavso_prefill', {}).get('planet')) or "HAT-P-32 b", + ) planet_label.grid(row=i, column=j, sticky=tk.W, pady=2) planet_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -1235,7 +1377,10 @@ def save_input(): # "Host Star Name": "HAT-P-32", star_label = tk.Label(root, text="Host Star Name", justify=tk.LEFT) star_entry = tk.Entry(root, font="Helvetica 12", justify=tk.LEFT) - star_entry.insert(tk.END, "HAT-P-32") + star_entry.insert( + tk.END, + stringify_prefill(input_data.get('aavso_prefill', {}).get('host_star')) or "HAT-P-32", + ) star_label.grid(row=i, column=j, sticky=tk.W, pady=2) star_entry.grid(row=i, column=j + 1, sticky=tk.W, pady=2) i += 1 @@ -1376,7 +1521,7 @@ def save_input(): "Comment4": "and is only here to serve as a guide. Will be updated per user's advice.", "Image Calibrations Directory Guide": "Enter in the path to image calibrations or enter in null for none.", "Planetary Parameters Guide": "For planetary parameters that are not filled in, enter in null.", - "Comparison Star(s) Guide": "Up to 10 comparison stars can be added following the format given below.", + "Comparison Star(s) Guide": "Provide comparison stars either as X/Y pixels or as RA/Dec coordinates, but not both. RA/Dec requires a usable reference-image WCS.", "Obs. Latitude Guide": "Indicate the sign (+ North, - South) before the degrees. Needs to be in decimal or HH:MM:SS format.", "Obs. Longitude Guide": "Indicate the sign (+ East, - West) before the degrees. Needs to be in decimal or HH:MM:SS format.", "Plate Solution": "For your image to be given a plate solution, type y.", @@ -1387,6 +1532,35 @@ def save_input(): "Target Star DEC": "Must be in +/-DD:MM:SS sexagesimal format with correct sign at the beginning (+ or -).", "Demosaic Format": "Optional control for handling Bayer pattern color images - to use, provide Bayer color patttern of your camera (RGGB, BGGR, GRBG, GBRG) - null (no color processing) is default", "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", + "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", + "Pixel Alignment Fallback": "Set optional_info 'allow_pixel_alignment_fallback' to false to require WCS-only processing and drop every frame without celestial WCS. Default true; EXOTIC prefers WCS when coverage is consistent and otherwise retains the sequence for legacy alignment.", + "Bad WCS Threshold Percent": "When allow_pixel_alignment_fallback is true, set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS while still using WCS-only processing. Below the threshold, missing-WCS frames are dropped; at or above it, all frames are retained for alignment fallback. Default 3.", + "Prefer Pixel Coordinates Over WCS": "Set optional_info 'prefer_pixel_values_over_wcs_for_target' to y to keep the entered target pixel coordinates when they conflict with WCS-derived target coordinates. Default n.", + "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", + "Stellar Variability Only": "Set optional_info 'stellar_variability_only' to true to skip transit fitting, select comparison-star photometry by out-of-transit scatter, and discard predicted ingress-to-egress transit-window points. Default false.", + "Apparent Magnitudes Required": "Set optional_info 'require_apparent_magnitudes' to false when catalogue-calibrated apparent magnitudes are not required. Differential-magnitude products remain independent of catalogue calibration. Stellar-variability magnitudes use the raw target/reference ratio with no airmass correction. Default true.", + "Use Exactly Supplied Comparisons": "Set optional_info 'use_exactly_the_comps_provided' to true to use only the supplied X/Y or RA/Dec comparison coordinates with no replacement, addition, vetting, ranking, or ensemble-size limit. One comparison is used alone; multiple comparisons are all used as a fixed ensemble. Default false.", + "Maximum Transit Ensemble Comparisons": "Set optional_info 'maximum_number_of_ensemble_comparisons_for_transit' to the largest number of comparison stars used by the transit-fit ensemble. Default 5; minimum 2; no configured upper limit.", + "Maximum Stellar-Variability Ensemble Comparisons": "Set optional_info 'maximum_number_of_ensemble_comparisons_for_stellar_variability' to the largest number of comparison stars used by stellar-variability-only and fortuitous-variable ensembles. Default 5; minimum 2; no configured upper limit.", + "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into working_artifacts/, and median-8 repair those pixels before centroiding and photometry. Default n.", + "Multiprocess Bad-Pixel Precheck": "Set optional_info 'multiprocess_bad_pixel_precheck' to y or a positive process count to scan bad pixels in parallel. Default n.", + "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", + "Final Fit Baseline Duration Multiplier": "Set optional_info 'final_fit_baseline_duration_multiplier' to the number of fitted transit durations to keep as baseline before ingress and after egress during the automatic final-fit prefit/refit. Default 1.0.", + "EEBLS Tmid Initializer": "Set optional_info 'use_eebls_to_initialize_tmid_and_bounds' to y to run a fixed-period box least squares search over the light curve, use the strongest bracketed transit-like signal to initialize Tmid, and narrow the Tmid search range before fitting. Default y.", + "Pick Comparison by EEBLS SNR": "Set optional_info 'pick_comparison_by_eebls_snr' to y to prefer the comparison star whose target light curve yields the highest finite EEBLS SNR, falling back to residual scatter if no usable EEBLS SNR is available. Default y.", + "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", + "Maximum Rp/Rs Search Bound": "Set optional_info 'rprs_search_bound_max' to cap the nested-fit Rp/Rs search range. Default 0.5.", + "Restrict Rp/Rs Search Range": "Set optional_info 'restrict_Rp/Rs_range' to y to restrict Rp/Rs to a prior-centered percentage window. Set 'restrict_Rp/Rs_range_percentage' to control the half-width. Defaults y and 10.", + "Prior Rp/Rs Fallback For Pinned Posterior": "Set optional_info 'use_prior_Rp/Rs_when_posterior_pinned' to y to rerun a fit with Rp/Rs fixed to the input prior and quote a data-only Rp/Rs uncertainty when the Rp/Rs posterior remains edge-pinned after retry handling. Default y.", + "Restrict a/Rs Search Range": "Set optional_info 'restrict_a/Rs_range' to y to restrict a/Rs to a prior-centered percentage window. Set 'restrict_a/Rs_range_percentage' to control the half-width. Defaults y and 10.", + "Sparse Posterior Live-Point Retry": "Set optional_info 'use_sparse_posterior_live_point_retry' to y to rank comparison-star candidates at the configured UltraNest live-point count, then continue the chosen final comparison-star fit with 5x additional minimum live points using its retained final-pass bounds. Standalone final fits still only continue when Rp/Rs, Tmid, or a/Rs posteriors are too sparse. Set to n to disable. Default y.", + "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", + "Reject Overexposed Stars": "Set optional_info 'reject_overexposed_stars' to true to reject overexposed target frames and overexposed comparison-star measurements. Default true.", + "Saturation Value": "Set optional_info 'saturation_value' to the detector saturation value in the same units as the image pixels. If omitted/default, EXOTIC uses FITS SATURATE when available, maps TELESCOP Cecilia to 4096, otherwise uses 65535.", + "Overexposure Threshold Fraction": "Set optional_info 'overexposure_threshold_fraction' to the fraction of saturation used for rejection. Default 0.9.", + "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require a real comparison star for the best-fit photometry result.", + "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", + "Boolean Values": "All boolean settings accept JSON true/false, numeric 1/0, or case-insensitive strings y/n. The equivalent strings yes/no and on/off are also accepted.", "Formatting of null": "Due to the file being a .json, null is case sensitive and must be spelled as shown.", "Decimal Format": "Leading zero must be included when appropriate (Ex: 0.32, .32 or 00.32 causes errors.)." } @@ -1404,6 +1578,7 @@ def save_input(): "AAVSO Observer Code (blank if none)": input_data['obscode'], "Secondary Observer Codes (blank if none)": input_data['secondobscode'], + "Observatory Full Title": input_data.get('obs_name', ""), "Observation date": input_data['obsdate'], "Obs. Latitude": input_data['lat'], @@ -1419,6 +1594,7 @@ def save_input(): "Target Star X & Y Pixel": (input_data['targetpos']), "Comparison Star(s) X & Y Pixel": (input_data['comppos']), + "Comparison Star(s) RA & Dec": null, "Demosaic Format": null, # TODO add GUI input for these "Demosaic Output": null @@ -1447,7 +1623,44 @@ def save_input(): new_inits['optional_info'] = { "Filter Minimum Wavelength (nm)": input_data.get('filtermin', null), - "Filter Maximum Wavelength (nm)": input_data.get('filtermax', null) + "Filter Maximum Wavelength (nm)": input_data.get('filtermax', null), + "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, + "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", + "allow_pixel_alignment_fallback": True, + "bad_wcs_threshold_percent": 3.0, + "prefer_pixel_values_over_wcs_for_target": "n", + "disable vertical flux normalization": False, + "stellar_variability_only": False, + "require_apparent_magnitudes": True, + "use_exactly_the_comps_provided": False, + "maximum_number_of_ensemble_comparisons_for_transit": 5, + "maximum_number_of_ensemble_comparisons_for_stellar_variability": 5, + "detect_bad_pixels_before_photometry": "n", + "multiprocess_bad_pixel_precheck": "n", + "detrend_on_outoftransit_baseline": True, + "final_fit_baseline_duration_multiplier": 1.0, + "use_eebls_to_initialize_tmid_and_bounds": "y", + "pick_comparison_by_eebls_snr": "y", + "use_impactparameter_rather_than_inclination_to_fit": "y", + "rprs_search_bound_max": 0.5, + "restrict_Rp/Rs_range": "y", + "restrict_Rp/Rs_range_percentage": 10.0, + "use_prior_Rp/Rs_when_posterior_pinned": "y", + "restrict_a/Rs_range": "y", + "restrict_a/Rs_range_percentage": 10.0, + "use_sparse_posterior_live_point_retry": "y", + "use_adaptive_apertures": False, + "reject_overexposed_stars": True, + "saturation_value": 65535, + "overexposure_threshold_fraction": 0.9, + "gain_electrons_per_adu": null, + "read_noise_electrons": null, + "dark_current_electrons_per_second_per_pixel": null, + "flat_field_fractional_error": null, + "telescope_aperture_m": null, + "scintillation_coefficient": null, + "Use target-driven comp selection rather than comp-driven comp selection": "n", + "require_comp_star": "y" } if 'pixscale' not in input_data.keys(): @@ -1465,11 +1678,12 @@ def save_input(): "AAVSO Observer Code (blank if none)": input_data['obscode'], "Secondary Observer Codes (blank if none)": input_data['secondobscode'], + "Observatory Full Title": input_data.get('obs_name', ""), "Observation date": input_data['obsdate'], "Obs. Latitude": input_data['lat'], "Obs. Longitude": input_data['long'], - "Obs. Elevation (meters)": float(input_data.get('elevation', 0)), + "Obs. Elevation (meters; Note: leave blank if unknown)": input_data.get('elevation'), "Camera Type (CCD or DSLR)": input_data['cameratype'], "Pixel Binning": input_data['pixbin'], "Filter Name (aavso.org/filters)": input_data['obsfilter'], @@ -1487,7 +1701,39 @@ def save_input(): "Pre-reduced File Time Format (BJD_TDB, JD_UTC, MJD_UTC)": input_data['file_time'], "Pre-reduced File Units of Flux (flux, magnitude, millimagnitude)": input_data['file_units'], "Comparison Star used in Photometry (blank if none)": input_data['phot_comp_star'], - "Exposure Time (s)": input_data['exp'] + "Exposure Time (s)": input_data['exp'], + "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, + "Ignore WCS in Header and Do Manual Alignment? (y/n)": "n", + "allow_pixel_alignment_fallback": True, + "bad_wcs_threshold_percent": 3.0, + "prefer_pixel_values_over_wcs_for_target": "n", + "disable vertical flux normalization": False, + "stellar_variability_only": False, + "require_apparent_magnitudes": True, + "use_exactly_the_comps_provided": False, + "maximum_number_of_ensemble_comparisons_for_transit": 5, + "maximum_number_of_ensemble_comparisons_for_stellar_variability": 5, + "detect_bad_pixels_before_photometry": "n", + "multiprocess_bad_pixel_precheck": "n", + "detrend_on_outoftransit_baseline": True, + "final_fit_baseline_duration_multiplier": 1.0, + "use_eebls_to_initialize_tmid_and_bounds": "y", + "pick_comparison_by_eebls_snr": "y", + "use_impactparameter_rather_than_inclination_to_fit": "y", + "use_prior_Rp/Rs_when_posterior_pinned": "y", + "use_sparse_posterior_live_point_retry": "y", + "use_adaptive_apertures": False, + "reject_overexposed_stars": True, + "saturation_value": 65535, + "overexposure_threshold_fraction": 0.9, + "gain_electrons_per_adu": null, + "read_noise_electrons": null, + "dark_current_electrons_per_second_per_pixel": null, + "flat_field_fractional_error": null, + "telescope_aperture_m": null, + "scintillation_coefficient": null, + "Use target-driven comp selection rather than comp-driven comp selection": "n", + "require_comp_star": "y" } if planetparams.get() in ["manual", "nea"]: @@ -1516,7 +1762,10 @@ def save_input(): "Star Metallicity (-) Uncertainty": float(input_data['metUncNeg']), "Star Surface Gravity (log(g))": float(input_data['logg']), "Star Surface Gravity (+) Uncertainty": float(input_data['loggUncPos']), - "Star Surface Gravity (-) Uncertainty": float(input_data['loggUncNeg']) + "Star Surface Gravity (-) Uncertainty": float(input_data['loggUncNeg']), + "Star Distance (pc)": null if input_data.get('dist') in (None, "") else float(input_data['dist']), + "Star Proper Motion RA (mas/yr)": null if input_data.get('pm_ra') in (None, "") else float(input_data['pm_ra']), + "Star Proper Motion DEC (mas/yr)": null if input_data.get('pm_dec') in (None, "") else float(input_data['pm_dec']) } elif planetparams.get() == "inits": diff --git a/exotic/inputs.py b/exotic/inputs.py index d4ad1a4c..67e07384 100644 --- a/exotic/inputs.py +++ b/exotic/inputs.py @@ -1,30 +1,254 @@ import logging import sys import json +import math from pathlib import Path +import requests from astropy.io import fits +from astropy.time import Time +from astropy.coordinates import SkyCoord +import astropy.units as u import re try: - from utils import user_input, init_params, typecast_check, \ + from utils import coerce_boolean_config_value, user_input, init_params, typecast_check, \ process_lat_long, find, open_elevation except ImportError: - from .utils import user_input, init_params, typecast_check, \ + from .utils import coerce_boolean_config_value, user_input, init_params, typecast_check, \ process_lat_long, find, open_elevation try: from animate import animate_toggle except ImportError: from .animate import animate_toggle +try: + from api.filters import fwhm as photometric_filters, fwhm_alias as photometric_filter_aliases +except ImportError: + from .api.filters import fwhm as photometric_filters, fwhm_alias as photometric_filter_aliases log = logging.getLogger(__name__) -logging.basicConfig(filename='exotic.log', level=logging.DEBUG) consoleFormatter = logging.Formatter("%(message)s") consoleHandler = logging.StreamHandler(sys.stdout) consoleHandler.setFormatter(consoleFormatter) consoleHandler.setLevel(logging.INFO) log.addHandler(consoleHandler) +PHOT_COMP_STAR_KEYS = ("ra", "dec", "x", "y") +AAVSO_OBSDATE_HEADER_KEYS = ('OBSDATE',) +AAVSO_LOCATION_HEADER_KEYS = { + 'lat': ('OBSLAT', 'LATITUDE', 'OBS_LATITUDE', 'LAT'), + 'long': ('OBSLON', 'OBSLONG', 'LONGITUDE', 'OBS_LONGITUDE', 'LONG'), + 'elev': ('OBSELEV', 'OBSALT', 'ELEVATION', 'ALTITUDE', 'HEIGHT'), +} +AAVSO_FILTER_HEADER_KEYS = ('FILTER',) +AAVSO_FILTER_XC_HEADER_KEYS = ('FILTER-XC',) +AAVSO_TEXT_HEADER_KEYS = { + 'aavso_num': ('OBSCODE',), + 'second_obs': ('SECONDARY_OBSCODES',), + 'obs_name': ('OBSNAME',), + 'camera': ('OBSTYPE',), + 'pixel_bin': ('BINNING',), + 'notes': ('NOTES',), + 'planet': ('EXOPLANET_NAME',), + 'host_star': ('STAR_NAME',), +} +AAVSO_GAIA_HEADER_KEYS = { + 'dist': ('GAIADIST',), + 'pm_ra': ('GAIAPMRA',), + 'pm_dec': ('GAIAPMDEC', 'GAIADEC'), +} +AAVSO_EXPOSURE_HEADER_KEYS = ('EXPOSURE_TIME', 'EXPTIME', 'EXPOSURE', 'EXP') +AAVSO_TIME_FORMAT_HEADER_KEYS = ('DATE_TYPE',) +AAVSO_MEASUREMENT_TYPE_HEADER_KEYS = ('MEASUREMENT_TYPE',) +AAVSO_DETREND_PARAMETER_HEADER_KEYS = ('DETREND_PARAMETERS',) +AAVSO_ALLOWED_FILE_TIME_FORMATS = {'BJD_TDB', 'JD_UTC', 'MJD_UTC'} +AAVSO_WAVELENGTH_UNIT_FACTORS_TO_NM = { + 'a': 0.1, + 'angstrom': 0.1, + 'angstroms': 0.1, + 'nm': 1.0, + 'nanometer': 1.0, + 'nanometers': 1.0, + 'um': 1000.0, + 'micron': 1000.0, + 'microns': 1000.0, + 'micrometer': 1000.0, + 'micrometers': 1000.0, + 'mum': 1000.0, +} +NEXTASTRO_GAIA_DISTPM_ENDPOINT = 'https://archive.nextastro.org/single_star_gaia_distpm' +NEXTASTRO_REQUEST_TIMEOUT = 30 + + +def is_blank_value(value): + if value is None: + return True + if isinstance(value, str): + return value.strip().lower() in ('', 'n/a', 'na', 'null', 'none') + return False + + +def normalize_aavso_filter_lookup_key(value): + if is_blank_value(value): + return None + return re.sub(r'[\W_]+', '', str(value).strip().lower()) + + +def coerce_finite_float(value): + if is_blank_value(value): + return None + + try: + numeric_value = float(str(value).strip()) + except (TypeError, ValueError): + return None + + if not math.isfinite(numeric_value): + return None + + return numeric_value + + +def radec_to_decimal_degrees(ra, dec): + if is_blank_value(ra) or is_blank_value(dec): + return None, None + + ra_value = str(ra).strip() + dec_value = str(dec).strip() + ra_unit = u.hourangle if any(separator in ra_value for separator in (':', ' ')) else u.deg + + if ra_unit is u.hourangle: + ra_value = ra_value.replace(':', ' ') + if any(separator in dec_value for separator in (':', ' ')): + dec_value = dec_value.replace(':', ' ') + + try: + coords = SkyCoord(ra=ra_value, dec=dec_value, unit=(ra_unit, u.deg)) + except ValueError: + return None, None + + if not math.isfinite(coords.ra.degree) or not math.isfinite(coords.dec.degree): + return None, None + + return coords.ra.degree, coords.dec.degree + + +def comparison_star_coords_provided(comp_stars): + """Return whether an X/Y comparison-star input contains a coordinate pair.""" + if isinstance(comp_stars, (list, tuple)): + if len(comp_stars) == 2 and not any(isinstance(value, (list, tuple, dict)) for value in comp_stars): + return not any(is_blank_value(value) for value in comp_stars) + return any( + isinstance(star, (list, tuple)) + and len(star) == 2 + and not any(is_blank_value(value) for value in star) + for star in comp_stars + ) + if isinstance(comp_stars, str): + return len(re.findall(r"[-+]?(?:\d*\.?\d+)", comp_stars)) >= 2 + return False + + +def comparison_star_radec_coords(comp_stars): + """Normalize one or more comparison-star RA/Dec pairs to decimal degrees.""" + if is_blank_value(comp_stars): + return [] + + if isinstance(comp_stars, str): + try: + comp_stars = json.loads(comp_stars) + except json.JSONDecodeError as exc: + raise ValueError( + "Comparison Star(s) RA & Dec must be a JSON list of [RA, Dec] pairs." + ) from exc + + if ( + isinstance(comp_stars, (list, tuple)) + and len(comp_stars) == 2 + and not any(isinstance(value, (list, tuple, dict)) for value in comp_stars) + ): + comp_stars = [comp_stars] + + if not isinstance(comp_stars, (list, tuple)): + raise ValueError("Comparison Star(s) RA & Dec must be a list of [RA, Dec] pairs.") + + normalized = [] + for index, star in enumerate(comp_stars, start=1): + if is_blank_value(star) or star == [] or star == (): + continue + if not isinstance(star, (list, tuple)) or len(star) != 2: + raise ValueError( + f"Comparison star {index} RA/Dec must contain exactly two values: [RA, Dec]." + ) + try: + ra_deg, dec_deg = radec_to_decimal_degrees(star[0], star[1]) + except (TypeError, ValueError) as exc: + raise ValueError(f"Comparison star {index} has invalid RA/Dec coordinates: {exc}") from exc + if ra_deg is None or dec_deg is None: + raise ValueError(f"Comparison star {index} has blank RA or Dec coordinates.") + normalized.append([float(ra_deg), float(dec_deg)]) + return normalized + + +def fetch_nextastro_gaia_distpm(ra_deg, dec_deg): + response = requests.get( + NEXTASTRO_GAIA_DISTPM_ENDPOINT, + params={'ra': ra_deg, 'dec': dec_deg}, + timeout=NEXTASTRO_REQUEST_TIMEOUT, + ) + response.raise_for_status() + + payload = response.json() + gaia = payload.get('gaia') if isinstance(payload, dict) else None + if not isinstance(gaia, dict): + return {} + + return { + 'dist': coerce_finite_float(gaia.get('distance_pc')), + 'pm_ra': coerce_finite_float(gaia.get('pmra_mas_per_year')), + 'pm_dec': coerce_finite_float(gaia.get('pmdec_mas_per_year')), + } + + +def populate_missing_gaia_astrometry(planet_dict): + missing_keys = [key for key in ('dist', 'pm_ra', 'pm_dec') if is_blank_value(planet_dict.get(key))] + if not missing_keys: + return planet_dict + + ra_deg, dec_deg = radec_to_decimal_degrees(planet_dict.get('ra'), planet_dict.get('dec')) + if ra_deg is None or dec_deg is None: + return planet_dict + + try: + gaia_values = fetch_nextastro_gaia_distpm(ra_deg, dec_deg) + except requests.exceptions.RequestException as exc: + log_info(f"\nWarning: NextAstro Gaia astrometry lookup failed ({exc}); continuing without missing Gaia values.", + warn=True) + return planet_dict + + filled_keys = [] + for key in missing_keys: + if gaia_values.get(key) is None: + continue + planet_dict[key] = gaia_values[key] + filled_keys.append(key) + + if filled_keys: + log_info("\nRetrieved missing Gaia distance/proper motion from NextAstro archive lookup.") + + return planet_dict + + +AAVSO_FILTER_LOOKUP = {} +for filter_desc, filter_metadata in photometric_filters.items(): + AAVSO_FILTER_LOOKUP[normalize_aavso_filter_lookup_key(filter_desc)] = filter_metadata + AAVSO_FILTER_LOOKUP[normalize_aavso_filter_lookup_key(filter_metadata.get('name'))] = filter_metadata + +for alias, canonical in photometric_filter_aliases.items(): + filter_metadata = photometric_filters.get(canonical) + if filter_metadata is not None: + AAVSO_FILTER_LOOKUP[normalize_aavso_filter_lookup_key(alias)] = filter_metadata + class Inputs: @@ -32,12 +256,63 @@ def __init__(self, init_opt): self.init_opt = init_opt self.info_dict = { 'images': None, 'save': None, 'flats': None, 'darks': None, 'biases': None, - 'aavso_num': None, 'second_obs': None, 'date': None, 'lat': None, 'long': None, + 'aavso_num': None, 'second_obs': None, 'obs_name': '', 'date': None, 'lat': None, 'long': None, 'elev': None, 'camera': None, 'pixel_bin': None, 'filter': None, 'notes': None, 'plate_opt': None, 'aavso_comp': None, 'tar_coords': None, 'comp_stars': None, + 'comp_stars_radec': None, 'prered_file': None, 'file_units': None, 'file_time': None, 'phot_comp_star': None, 'wl_min': None, 'wl_max': None, 'pixel_scale': None, 'exposure': None, - 'random_seed': None, "demosaic_fmt": None, "demosaic_out": None + 'dist': None, 'pm_ra': None, 'pm_dec': None, 'airmass_already_corrected': False, + 'random_seed': None, 'ld_uncertainties': None, "demosaic_fmt": None, "demosaic_out": None, + 'fast_aperture_mask': False, 'require_comp_star': 'y', 'ignore_header_wcs': 'n', + 'allow_pixel_alignment_fallback': True, + 'prefer_pixel_values_over_wcs_for_target': 'n', + 'target_driven_comp_selection': 'n', 'disable_vertical_flux_normalization': False, + 'stellar_variability_only': False, + 'use_ensemble_photometry_for_stellar_variability': True, + 'require_apparent_magnitudes': True, + 'use_exactly_the_comps_provided': False, + 'maximum_number_of_ensemble_comparisons_for_transit': 5, + 'maximum_number_of_ensemble_comparisons_for_stellar_variability': 5, + 'photometer_fortuitous_variables': True, + 'use_single_comparison_for_fortuitous_variables': True, + 'use_nextastro_vsx_cache_first': False, + 'detrend_on_outoftransit_baseline': True, + 'final_fit_baseline_duration_multiplier': 1.0, + 'use_eebls_to_initialize_tmid_and_bounds': 'y', + 'pick_comparison_by_eebls_snr': 'y', + 'use_deviation_from_expected_transit_in_qc': True, + 'deviation_from_expected_transit_in_qc_sigma': 5.0, + 'run_final_fit_phase_residual_clip': 'y', + 'exit_at_first_qc_pass_solution': 'y', + 'detect_bad_pixels_before_photometry': 'n', + 'multiprocess_bad_pixel_precheck': 'n', + 'use_impactparameter_rather_than_inclination_to_fit': 'y', + 'use_psf_photometry': 'y', 'use_aperture_photometry': 'y', + 'use_legacy_psf_flux': 'n', + 'psf_seed_track_directory': None, + 'use_adaptive_apertures': False, 'bad_wcs_threshold_percent': 3.0, + 'use_aperture_corrections_and_full_image_fwhm': False, + 'pointing_rejection_sigma': None, + 'reject_overexposed_stars': True, + 'saturation_value': 65535.0, + 'overexposure_threshold_fraction': 0.9, + 'gain_electrons_per_adu': None, + 'read_noise_electrons': None, + 'dark_current_electrons_per_second_per_pixel': None, + 'flat_field_fractional_error': None, + 'telescope_aperture_m': None, + 'scintillation_coefficient': None, + 'skip_low_comparison_coverage_rejection': 'n', + 'fit_lightcurve_to_every_comparison_candidate': 'n', + 'ultranest_min_num_live_points': 200, + 'rprs_search_bound_max': 0.5, + 'restrict_rprs_range': 'y', + 'restrict_rprs_range_percentage': 10.0, + 'use_prior_rprs_when_posterior_pinned': 'y', + 'restrict_ars_range': 'y', + 'restrict_ars_range_percentage': 10.0, + 'use_sparse_posterior_live_point_retry': 'y', } self.params = { 'images': imaging_files, 'save': save_directory, 'aavso_num': obs_code, 'second_obs': second_obs_code, @@ -62,11 +337,18 @@ def complete_red(self, planet): elif key == 'tar_coords': self.info_dict[key] = self.params[key](self.info_dict[key], planet) elif key == 'comp_stars': - self.info_dict[key] = self.params[key](self.info_dict[key], False) + if self.info_dict.get('comp_stars_radec'): + # Celestial comparison coordinates are projected into pixels + # after the reference image has a usable WCS. + self.info_dict[key] = [] + else: + self.info_dict[key] = self.params[key](self.info_dict[key], False) elif key == 'images': pass elif key in ('lat', 'long'): self.info_dict[key] = self.params[key](self.info_dict[key], hdr) + elif key == 'pixel_bin': + self.info_dict[key] = self.params[key](self.info_dict[key], hdr) else: self.info_dict[key] = self.params[key](self.info_dict[key]) if key == 'save': @@ -81,23 +363,50 @@ def complete_red(self, planet): return self.info_dict, planet def prereduced(self, planet): - rem_list = ['images', 'plate_opt', 'tar_coords', 'comp_stars'] + rem_list = ['images', 'plate_opt', 'aavso_comp', 'tar_coords', 'comp_stars'] [self.params.pop(key) for key in rem_list] + self.info_dict['aavso_comp'] = 'n' self.params.update({'exposure': exposure, 'file_units': data_file_units, 'file_time': data_file_time, 'phot_comp_star': phot_comp_star}) self.info_dict['prered_file'] = prereduced_file(self.info_dict['prered_file']) - + aavso_overrides = parse_aavso_prereduced_overrides(self.info_dict['prered_file']) + + for key in ( + 'aavso_num', 'second_obs', 'obs_name', 'lat', 'long', 'elev', 'camera', 'pixel_bin', + 'filter', 'notes', 'wl_min', 'wl_max', 'exposure', 'file_time', 'file_units', + 'dist', 'pm_ra', 'pm_dec' + ): + if is_blank_value(self.info_dict.get(key)) and aavso_overrides.get(key) is not None: + self.info_dict[key] = aavso_overrides[key] + self.info_dict['airmass_already_corrected'] = bool(aavso_overrides.get('airmass_already_corrected')) + + if not planet and not is_blank_value(aavso_overrides.get('planet')): + planet = aavso_overrides['planet'] if not planet: planet = planet_name(planet) for key, value in list(self.params.items()): if key == 'elev': self.info_dict[key] = self.params[key](self.info_dict[key], self.info_dict['lat'], - self.info_dict['long']) + self.info_dict['long'], required=False) + elif key == 'lat': + self.info_dict[key] = self.params[key](self.info_dict[key], required=False) + elif key == 'long': + self.info_dict[key] = self.params[key](self.info_dict[key], required=False) + elif key == 'phot_comp_star': + self.info_dict[key] = self.params[key](self.info_dict[key], self.info_dict['prered_file']) + elif key == 'date': + continue else: self.info_dict[key] = self.params[key](self.info_dict[key]) + self.info_dict['date'] = prereduced_obs_date( + self.info_dict.get('date'), + self.info_dict['prered_file'], + self.info_dict.get('file_time'), + ) + return self.info_dict, planet def real_time(self, planet): @@ -158,29 +467,60 @@ def comp_params(self, init_file, planet_dict): 'demosaic_fmt': 'Demosaic Format', 'demosaic_out': 'Demosaic Output', 'aavso_num': ('AAVSO Observer Code (N/A if none)', 'AAVSO Observer Code (blank if none)'), 'second_obs': ('Secondary Observer Codes (N/A if none)', 'Secondary Observer Codes (blank if none)'), + 'obs_name': 'Observatory Full Title', 'date': 'Observation date', 'lat': 'Obs. Latitude', 'long': 'Obs. Longitude', 'elev': ('Obs. Elevation (meters)', 'Obs. Elevation (meters; Note: leave blank if unknown)'), - 'camera': 'Camera Type (CCD or DSLR)', + 'camera': ( + 'Camera Type (CCD or DSLR)', + 'Camera Type', + 'Camera Type (e.g., CCD or DSLR)', + 'Camera Type (e.g., CCD or DSLR; Note: if you are using a CMOS, please enter CCD here and then note your actual camera type in "Observing Notes")' + ), 'pixel_bin': 'Pixel Binning', 'filter': 'Filter Name (aavso.org/filters)', 'notes': 'Observing Notes', 'plate_opt': 'Plate Solution? (y/n)', 'aavso_comp': 'Add Comparison Stars from AAVSO? (y/n)', - 'tar_coords': 'Target Star X & Y Pixel', 'comp_stars': 'Comparison Star(s) X & Y Pixel', + 'tar_coords': 'Target Star X & Y Pixel', + 'comp_stars': 'Comparison Star(s) X & Y Pixel', + 'comp_stars_radec': ( + 'Comparison Star(s) RA & Dec', + 'Comparison Star(s) RA and Dec', + 'Comparison Star(s) RA & Dec (degrees)', + ), + # Accept existing init files that placed this option beside the + # comparison coordinates. optional_info is parsed later and wins + # when both locations are populated. + 'use_exactly_the_comps_provided': 'use_exactly_the_comps_provided', } planet_params = { 'ra': 'Target Star RA', 'dec': 'Target Star Dec', 'pName': "Planet Name", 'sName': "Host Star Name", 'pPer': 'Orbital Period (days)', 'pPerUnc': 'Orbital Period Uncertainty', - 'midT': 'Published Mid-Transit Time (BJD-UTC)', 'midTUnc': 'Mid-Transit Time Uncertainty', - 'rprs': 'Ratio of Planet to Stellar Radius (Rp/Rs)', - 'rprsUnc': 'Ratio of Planet to Stellar Radius (Rp/Rs) Uncertainty', - 'aRs': 'Ratio of Distance to Stellar Radius (a/Rs)', - 'aRsUnc': 'Ratio of Distance to Stellar Radius (a/Rs) Uncertainty', + 'midT': ('Published Mid-Transit Time (BJD-UTC)', 'Published Mid-Transit Time'), + 'midTUnc': 'Mid-Transit Time Uncertainty', + 'rprs': ('Ratio of Planet to Stellar Radius (Rp/Rs)', 'Rp/Rs', 'Rp/R*'), + 'rprsUnc': ( + 'Ratio of Planet to Stellar Radius (Rp/Rs) Uncertainty', + 'Rp/Rs Uncertainty', + 'Rp/R* Uncertainty', + ), + 'aRs': ('Ratio of Distance to Stellar Radius (a/Rs)', 'a/Rs', 'a/R*'), + 'aRsUnc': ( + 'Ratio of Distance to Stellar Radius (a/Rs) Uncertainty', + 'a/Rs Uncertainty', + 'a/R* Uncertainty', + ), 'inc': 'Orbital Inclination (deg)', - 'incUnc': ('Orbital Inclination (deg) Uncertainty', 'Orbital Inclination (deg) Uncertainity'), - 'ecc': 'Orbital Eccentricity (0 if null)', 'teff': 'Star Effective Temperature (K)', + 'incUnc': ( + 'Orbital Inclination (deg) Uncertainty', + 'Orbital Inclination (deg) Uncertainity', + 'Orbital Inclination Uncertainty', + ), + 'ecc': ('Orbital Eccentricity (0 if null)', 'Orbital Eccentricity'), + 'teff': 'Star Effective Temperature (K)', 'omega': 'Argument of Periastron (deg)', 'teffUncPos': 'Star Effective Temperature (+) Uncertainty', 'teffUncNeg': 'Star Effective Temperature (-) Uncertainty', - 'met': 'Star Metallicity ([FE/H])', 'metUncPos': 'Star Metallicity (+) Uncertainty', + 'met': ('Star Metallicity ([FE/H])', 'Star Metallicity [FE/H]'), + 'metUncPos': 'Star Metallicity (+) Uncertainty', 'metUncNeg': 'Star Metallicity (-) Uncertainty', 'logg': 'Star Surface Gravity (log(g))', 'loggUncPos': 'Star Surface Gravity (+) Uncertainty', 'loggUncNeg': 'Star Surface Gravity (-) Uncertainty', @@ -191,16 +531,372 @@ def comp_params(self, init_file, planet_dict): opt_info = { 'prered_file': 'Pre-reduced File:', 'file_time': 'Pre-reduced File Time Format (BJD_TDB, JD_UTC, MJD_UTC)', 'file_units': 'Pre-reduced File Units of Flux (flux, magnitude, millimagnitude)', - 'phot_comp_star': "Comparison Star used in Photometry (leave blank if none)", + 'phot_comp_star': ( + "Comparison Star used in Photometry (leave blank if none)", + "Comparison Star used in Photometry (blank if none)" + ), 'wl_min': 'Filter Minimum Wavelength (nm)', 'wl_max': 'Filter Maximum Wavelength (nm)', - 'pixel_scale': ('Image Scale (Ex: 5.21 arcsecs/pixel)', 'Pixel Scale (Ex: 5.21 arcsecs/pixel)'), + 'ld_uncertainties': 'Calculate Limb Darkening Coefficients with Uncertainties? (y/n)', + 'fast_aperture_mask': ('Fast Aperture Mask (y/n)', 'Use Fast Aperture Mask (y/n)'), + 'require_comp_star': ('require_comp_star', 'Require Comparison Star? (y/n)'), + 'target_driven_comp_selection': ( + 'Use target-driven comp selection rather than comp-driven comp selection', + 'target_driven_comp_selection', + ), + 'ignore_header_wcs': ( + 'Ignore WCS in Header and Do Manual Alignment? (y/n)', + 'Ignore WCS in Header and Do Manual Alignment', + 'Ignore WCS in header and do manual alignment', + 'ignore_header_wcs', + ), + 'allow_pixel_alignment_fallback': ( + 'allow_pixel_alignment_fallback', + 'Allow Pixel Alignment Fallback', + 'Allow Pixel Alignment Fallback? (y/n)', + ), + 'prefer_pixel_values_over_wcs_for_target': ( + 'prefer_pixel_values_over_wcs_for_target', + 'Prefer Pixel Coordinates to WCS Coordinates if there is a conflict', + 'Prefer Pixel Coordinates to WCS Coordinates if there is a conflict? (y/n)', + ), + 'disable_vertical_flux_normalization': ( + 'disable vertical flux normalization', + 'Disable vertical flux normalization', + ), + 'stellar_variability_only': ( + 'stellar_variability_only', + 'stellar variability only', + 'Stellar Variability Only', + 'Stellar Variability Only? (y/n)', + ), + 'use_ensemble_photometry_for_stellar_variability': ( + 'use_ensemble_photometry_for_stellar_variability', + 'stellar_variability_use_ensemble', + 'Use Ensemble Photometry for Stellar Variability? (y/n)', + ), + 'require_apparent_magnitudes': ( + 'require_apparent_magnitudes', + 'Require Apparent Magnitudes? (y/n)', + ), + 'use_exactly_the_comps_provided': ( + 'use_exactly_the_comps_provided', + 'Use Exactly the Comparisons Provided? (y/n)', + ), + 'maximum_number_of_ensemble_comparisons_for_transit': ( + 'maximum_number_of_ensemble_comparisons_for_transit', + 'Maximum Number of Ensemble Comparisons for Transit', + ), + 'maximum_number_of_ensemble_comparisons_for_stellar_variability': ( + 'maximum_number_of_ensemble_comparisons_for_stellar_variability', + 'Maximum Number of Ensemble Comparisons for Stellar Variability', + ), + 'photometer_fortuitous_variables': ( + 'photometer_fortuitous_variables', + 'Photometer Fortuitous Variables? (y/n)', + ), + 'use_single_comparison_for_fortuitous_variables': ( + 'use_single_comparison_for_fortuitous_variables', + 'Use Single Comparison for Fortuitous Variables? (y/n)', + ), + 'use_nextastro_vsx_cache_first': ( + 'use_nextastro_vsx_cache_first', + 'Use NextAstro VSX Cache First? (y/n)', + ), + 'detect_bad_pixels_before_photometry': ( + 'detect_bad_pixels_before_photometry', + 'Detect Bad Pixels Before Photometry? (y/n)', + ), + 'multiprocess_bad_pixel_precheck': ( + 'multiprocess_bad_pixel_precheck', + 'Multiprocess Bad-Pixel Precheck? (y/n or process count)', + 'Multiprocess Bad Pixel Precheck? (y/n or process count)', + ), + 'detrend_on_outoftransit_baseline': ( + 'detrend_on_outoftransit_baseline', + 'Detrend on Out-of-Transit Baseline', + 'detrend_on_out_of_transit_baseline', + ), + 'final_fit_baseline_duration_multiplier': ( + 'final_fit_baseline_duration_multiplier', + 'Final Fit Baseline Duration Multiplier', + ), + 'use_eebls_to_initialize_tmid_and_bounds': ( + 'use_eebls_to_initialize_tmid_and_bounds', + 'Use EEBLS to Initialize Tmid and Bounds? (y/n)', + 'Use EEBLS To Initialize Tmid And Bounds? (y/n)', + ), + 'pick_comparison_by_eebls_snr': ( + 'pick_comparison_by_eebls_snr', + 'Pick Comparison by EEBLS SNR? (y/n)', + 'Pick comparison by EEBLS SNR? (y/n)', + ), + 'use_deviation_from_expected_transit_in_qc': ( + 'use_deviation_from_expected_transit_in_qc', + 'Use Deviation From Expected Transit In QC? (y/n)', + ), + 'deviation_from_expected_transit_in_qc_sigma': ( + 'deviation_from_expected_transit_in_qc_sigma', + 'Deviation From Expected Transit In QC Sigma', + ), + 'run_final_fit_phase_residual_clip': ( + 'run_final_fit_phase_residual_clip', + 'Run Final-Fit Phase Residual Clip? (y/n)', + 'Run Final Fit Phase Residual Clip? (y/n)', + ), + 'exit_at_first_qc_pass_solution': ( + 'exit_at_first_qc_pass_solution', + 'exit at first QC PASS solution', + 'Exit at first QC PASS solution', + 'Exit at first QC PASS solution? (y/n)', + 'Exit At First QC PASS Solution? (y/n)', + ), + 'use_impactparameter_rather_than_inclination_to_fit': ( + 'use_impactparameter_rather_than_inclination_to_fit', + 'Use impact parameter rather than inclination to fit? (y/n)', + 'Use Impact Parameter Rather Than Inclination To Fit? (y/n)', + ), + 'use_psf_photometry': ( + 'use_psf_photometry', + 'Use PSF Photometry? (y/n)', + ), + 'use_legacy_psf_flux': ( + 'use_legacy_psf_flux', + 'legacy_psf_flux_mode', + 'Use Legacy PSF Flux? (y/n)', + 'Use Legacy PSF Flux Mode? (y/n)', + ), + 'psf_seed_track_directory': ( + 'psf_seed_track_directory', + 'legacy_psf_seed_track_directory', + 'PSF Seed Track Directory', + 'Legacy PSF Seed Track Directory', + ), + 'use_aperture_photometry': ( + 'use_aperture_photometry', + 'Use Aperture Photometry? (y/n)', + ), + 'use_adaptive_apertures': ( + 'use_adaptive_apertures', + 'Use Adaptive Apertures? (y/n)', + 'Use Adaptive Apertures (y/n)', + ), + 'use_aperture_corrections_and_full_image_fwhm': ( + 'use_aperture_corrections_and_full_image_fwhm', + 'Use Aperture Corrections and Full Image FWHM? (y/n)', + 'Use Aperture Corrections And Full Image FWHM? (y/n)', + ), + 'reject_overexposed_stars': ( + 'reject_overexposed_stars', + 'Reject Overexposed Stars? (y/n)', + 'Reject Overexposed Target and Comparison Stars? (y/n)', + ), + 'saturation_value': ( + 'saturation_value', + 'saturation_value_adu', + 'Saturation Value', + 'SATURATE', + ), + 'overexposure_threshold_fraction': ( + 'overexposure_threshold_fraction', + 'Overexposure Threshold Fraction', + 'Saturation Rejection Threshold Fraction', + ), + 'skip_low_comparison_coverage_rejection': ( + 'skip_low_comparison_coverage_rejection', + 'Skip Low Comparison Coverage Rejection? (y/n)', + ), + 'fit_lightcurve_to_every_comparison_candidate': ( + 'fit_lightcurve_to_every_comparison_candidate', + 'Fit Lightcurve to Every Comparison Candidate? (y/n)', + ), + 'automatic_optimal_calibration_selector': ( + 'automatic_optimal_calibration_selector', + 'Automatic Optimal Calibration Selector? (y/n)', + ), + 'automatic_optimal_calibration_selector_count': ( + 'automatic_optimal_calibration_selector_count', + 'Automatic Optimal Calibration Selector Count', + 'automatic_optimal_calibration_selector_max_stars', + 'Automatic Optimal Calibration Selector Max Stars', + ), + 'colour_term': ( + 'colour_term', + 'color_term', + 'COLTERM', + ), + 'colour_term_error': ( + 'colour_term_error', + 'color_term_error', + 'COLTERR', + ), + 'colour_term_index': ( + 'colour_term_index', + 'color_term_index', + 'COLTIDX', + ), + 'colour_term_bv': ( + 'colour_term_bv', + 'color_term_bv', + 'COLTBV', + ), + 'colour_term_bv_error': ( + 'colour_term_bv_error', + 'color_term_bv_error', + 'COLTBVER', + 'COLTBVERR', + ), + 'colour_term_bprp': ( + 'colour_term_bprp', + 'color_term_bprp', + 'COLTBPRP', + ), + 'colour_term_bprp_error': ( + 'colour_term_bprp_error', + 'color_term_bprp_error', + 'CBPRPERR', + 'COLTBPRPERR', + ), + 'colour_equation_filter': ( + 'colour_equation_filter', + 'color_equation_filter', + 'COLEQFIL', + ), + 'use_ensemble_photometry_rather_than_single_comp': ( + 'use_ensemble_photometry_rather_than_single_comp', + 'Use Ensemble Photometry Rather Than Single Comp? (y/n)', + ), + 'ultranest_min_num_live_points': ( + 'Minimum Number of Live Points for UltraNest', + 'minimum number of live points for ultranest', + 'ultranest_min_num_live_points', + 'ultranest_min_live_points', + 'min_num_live_points', + ), + 'run_fast_ultranest_before_final_run': ( + 'run fast ultranest before final run', + 'Run Fast UltraNest Before Final Run? (y/n)', + 'run_fast_ultranest_before_final_run', + ), + 'rprs_search_bound_max': ( + 'rprs_search_bound_max', + 'max_rprs_search_bound', + 'maximum rprs search bound', + 'maximum Rp/Rs search bound', + 'maximum Rp/R* search bound', + 'Maximum Rp/Rs Search Bound', + 'Maximum Rp/R* Search Bound', + ), + 'restrict_rprs_range': ( + 'restrict_Rp/Rs_range', + 'restrict_Rp/R*_range', + 'restrict_rprs_range', + 'restrict_RpRs_range', + 'Restrict Rp/Rs Range? (y/n)', + 'Restrict Rp/R* Range? (y/n)', + ), + 'restrict_rprs_range_percentage': ( + 'restrict_Rp/Rs_range_percentage', + 'restrict_Rp/R*_range_percentage', + 'restrict_rprs_range_percentage', + 'restrict_RpRs_range_percentage', + 'Restrict Rp/Rs Range Percentage', + 'Restrict Rp/R* Range Percentage', + ), + 'use_prior_rprs_when_posterior_pinned': ( + 'use_prior_Rp/Rs_when_posterior_pinned', + 'use_prior_Rp/R*_when_posterior_pinned', + 'use_prior_rprs_when_posterior_pinned', + 'use_prior_RpRs_when_posterior_pinned', + 'Use Prior Rp/Rs When Posterior Pinned? (y/n)', + 'Use Prior Rp/R* When Posterior Pinned? (y/n)', + ), + 'restrict_ars_range': ( + 'restrict_a/Rs_range', + 'restrict_a/R*_range', + 'restrict_ars_range', + 'restrict_aRs_range', + 'Restrict a/Rs Range? (y/n)', + 'Restrict a/R* Range? (y/n)', + ), + 'restrict_ars_range_percentage': ( + 'restrict_a/Rs_range_percentage', + 'restrict_a/R*_range_percentage', + 'restrict_ars_range_percentage', + 'restrict_aRs_range_percentage', + 'Restrict a/Rs Range Percentage', + 'Restrict a/R* Range Percentage', + ), + 'use_sparse_posterior_live_point_retry': ( + 'use_sparse_posterior_live_point_retry', + 'Use Sparse Posterior Live-Point Retry? (y/n)', + 'Use Sparse Posterior Live Point Retry? (y/n)', + 'Sparse Posterior Live-Point Retry? (y/n)', + ), + 'bad_wcs_threshold_percent': ( + 'bad_wcs_threshold_percent', + 'Bad WCS Threshold Percent', + ), + 'pointing_rejection_sigma': ( + 'pointing_rejection_sigma', + 'Pointing Rejection Sigma', + ), + 'gain_electrons_per_adu': ( + 'gain_electrons_per_adu', + 'gain_e_per_adu', + 'Gain (e-/ADU)', + 'CCD Gain (e-/ADU)', + ), + 'read_noise_electrons': ( + 'read_noise_electrons', + 'read_noise_e', + 'read_noise', + 'Read Noise (e-)', + 'CCD Read Noise (e-)', + ), + 'dark_current_electrons_per_second_per_pixel': ( + 'dark_current_electrons_per_second_per_pixel', + 'dark_current_e_per_s_pix', + 'dark_current', + 'Dark Current (e-/s/pix)', + ), + 'flat_field_fractional_error': ( + 'flat_field_fractional_error', + 'flat_field_fractional_noise', + 'flat_field_error_fraction', + 'Flat Field Fractional Error', + 'Flat-Field Fractional Error', + ), + 'telescope_aperture_m': ( + 'telescope_aperture_m', + 'telescope_aperture_meters', + 'Telescope Aperture (m)', + ), + 'scintillation_coefficient': ( + 'scintillation_coefficient', + 'scintillation_noise_coefficient', + 'Scintillation Coefficient', + ), + 'pixel_scale': ('Image Scale (Ex: 5.21 arcsecs/pixel)', 'Pixel Scale (Ex: 5.21 arcsecs/pixel)', + 'Pixel Scale (arsec/pixel)'), 'exposure': 'Exposure Time (s)', 'random_seed': 'Random Seed' } self.info_dict = init_params(user_info, self.info_dict, data['user_info']) + self.info_dict['comp_stars_radec'] = comparison_star_radec_coords( + self.info_dict.get('comp_stars_radec') + ) + if self.info_dict['comp_stars_radec'] and comparison_star_coords_provided( + self.info_dict.get('comp_stars')): + raise ValueError( + "Provide comparison stars using either 'Comparison Star(s) X & Y Pixel' " + "or 'Comparison Star(s) RA & Dec', not both." + ) + if self.info_dict['aavso_comp'] is None: + self.info_dict['aavso_comp'] = 'n' self.info_dict = init_params(opt_info, self.info_dict, data['optional_info']) - return init_params(planet_params, planet_dict, data['planetary_parameters']) + planet_dict = init_params(planet_params, planet_dict, data['planetary_parameters']) + return populate_missing_gaia_astrometry(planet_dict) def check_imaging_files(directory, img_type): @@ -361,17 +1057,31 @@ def obs_date(date): return date -def latitude(lat, hdr=None): +def normalize_obs_date(date): + if is_blank_value(date): + return None + + date = str(date).strip() + if re.fullmatch(r'\d{8}', date): + date = f"{date[0:4]}-{date[4:6]}-{date[6:8]}" + if '/' in date: + date = date.replace('/', '-') + return date + + +def latitude(lat, hdr=None, required=True): while True: - if not lat: + if is_blank_value(lat): if hdr: lat = find(hdr, ['LATITUDE', 'LAT', 'SITELAT']) if lat: return lat + if not required: + return None lat = user_input("Enter the latitude (in degrees) of where you observed. " "(Don't forget the sign where North is '+' and South is '-')! " "(Example: -32.12): ", type_=str) - lat = lat.strip() + lat = str(lat).strip() if lat[0] == '+' or lat[0] == '-': # Convert to float if latitude in decimal. If latitude is in +/-HH:MM:SS format, convert to a float. @@ -391,17 +1101,19 @@ def latitude(lat, hdr=None): lat = None -def longitude(long, hdr=None): +def longitude(long, hdr=None, required=True): while True: - if not long: + if is_blank_value(long): if hdr: long = find(hdr, ['LONGITUD', 'LONG', 'LONGITUDE', 'SITELONG']) if long: return long + if not required: + return None long = user_input("Enter the longitude (in degrees) of where you observed. " "(Don't forget the sign where East is '+' and West is '-')! " "(Example: +152.51): ", type_=str) - long = long.strip() + long = str(long).strip() if long[0] == '+' or long[0] == '-': # Convert to float if longitude in decimal. If longitude is in +/-HH:MM:SS format, convert to a float. @@ -421,21 +1133,29 @@ def longitude(long, hdr=None): long = None -def elevation(elev, lat, long, hdr=None): +def elevation(elev, lat, long, hdr=None, required=True): while True: try: - elev = typecast_check(type_=float, val=elev) - if not elev: + if is_blank_value(elev): + elev = None + else: + elev = typecast_check(type_=float, val=elev) + if elev is False: + raise ValueError + + if elev is None: if hdr: elev = find(hdr, ['HEIGHT', 'ELEVATION', 'ELE', 'EL', 'OBSGEO-H', 'ALT-OBS', 'SITEELEV']) - if elev: - return int(elev) + if not is_blank_value(elev): + return float(elev) + if not required: + return None log_info("\nEXOTIC is retrieving elevation based on entered " "latitude and longitude from Open Elevation.") animate_toggle(True) elev = open_elevation(lat, long) animate_toggle() - if not elev: + if elev is False: log_info("\nWarning: EXOTIC could not retrieve elevation.", warn=True) elev = user_input("Enter the elevation (in meters) of where you observed: ", type_=float) return elev @@ -445,20 +1165,42 @@ def elevation(elev, lat, long, hdr=None): def camera(c_type): - while True: - if not c_type: - c_type = user_input("\nPlease enter the camera type (e.g., CCD or DSLR;\n" - "Note: if you are using a CMOS, please enter CCD here and\n" - "then note your actual camera type in \"Observing Notes\"): ", type_=str) - c_type = c_type.strip().upper() - if c_type not in ["CCD", "DSLR"]: - c_type = None - else: - return c_type + if isinstance(c_type, str) and "DSLR" in c_type.strip().upper(): + return "DSLR" + return "CCD" + + +def format_fits_binning_axis(value): + if is_blank_value(value): + return None + + try: + binning_value = float(str(value).strip()) + except (TypeError, ValueError): + return None + + if not math.isfinite(binning_value) or binning_value <= 0: + return None + if binning_value.is_integer(): + return str(int(binning_value)) + return str(binning_value) + +def fits_header_pixel_bin(hdr): + if hdr is None: + return None -def pixel_bin(pix_bin): - if not pix_bin: + x_binning = format_fits_binning_axis(find(hdr, ['XBINNING'])) + y_binning = format_fits_binning_axis(find(hdr, ['YBINNING'])) + if x_binning is None or y_binning is None: + return None + return f"{x_binning}x{y_binning}" + + +def pixel_bin(pix_bin, hdr=None): + if is_blank_value(pix_bin): + pix_bin = fits_header_pixel_bin(hdr) + if is_blank_value(pix_bin): pix_bin = user_input("Please enter the pixel binning: ", type_=str) return pix_bin @@ -473,23 +1215,23 @@ def obs_notes(notes): def plate_solution_opt(opt): - if opt: - opt = opt.lower().strip() - if opt not in ('y', 'n'): + parsed = None if is_blank_value(opt) else coerce_boolean_config_value(opt) + if parsed is None: opt = user_input("\nWould you like to upload the your image for a plate solution?" "\nThis will allow EXOTIC to translate your image's pixels into coordinates on the sky." "\nDISCLAIMER: One of your imaging files will be publicly viewable on " "nova.astrometry.net. (y/n): ", type_=str, values=['y', 'n']) - return opt + parsed = coerce_boolean_config_value(opt) + return 'y' if parsed else 'n' def aavso_comp(opt): - if opt: - opt = opt.lower().strip() - if opt not in ('y', 'n'): + parsed = None if is_blank_value(opt) else coerce_boolean_config_value(opt) + if parsed is None: opt = user_input("\nWould you like Comparison Stars added automatically from AAVSO? (y/n): ", type_=str, values=['y', 'n']) - return opt + parsed = coerce_boolean_config_value(opt) + return 'y' if parsed else 'n' def target_star_coords(coords, planet): @@ -506,7 +1248,7 @@ def target_star_coords(coords, planet): def comparison_star_coords(comp_stars, rt_bool): - if isinstance(comp_stars, list) and 1 <= len(comp_stars) <= 10 and \ + if isinstance(comp_stars, list) and len(comp_stars) >= 1 and \ all(isinstance(star, list) for star in comp_stars): comp_stars = [star for star in comp_stars if star != []] elif isinstance(comp_stars, str) and any(str.isdigit(x) for x in comp_stars): @@ -519,8 +1261,8 @@ def comparison_star_coords(comp_stars, rt_bool): if not comp_stars: while True: if not rt_bool: - num_comp_stars = user_input("\nHow many Comparison Stars would you like to use? (1-10): ", type_=int) - if 1 <= num_comp_stars <= 10: + num_comp_stars = user_input("\nHow many Comparison Stars would you like to use? (1 or more): ", type_=int) + if num_comp_stars >= 1: break log_info("\nError: The number of Comparison Stars entered is incorrect.", error=True) else: @@ -566,19 +1308,399 @@ def prereduced_file(file): file = None -def phot_comp_star(comp_star): +def blank_phot_comp_star(): + return {key: '' for key in PHOT_COMP_STAR_KEYS} + + +def normalize_phot_comp_star(comp_star): + normalized_comp_star = blank_phot_comp_star() + if not isinstance(comp_star, dict): - comp_star_opt = user_input("Was a Comparison Star used during Photometry? (y/n): ", - type_=str, values=['y', 'n']) - - comp_star = { - 'ra': user_input("\nEnter Comparison Star RA: ", type_=str) if comp_star_opt == 'y' else '', - 'dec': user_input("Enter Comparison Star DEC: ", type_=str) if comp_star_opt == 'y' else '', - 'x': user_input("\nEnter Comparison Star X Pixel Coordinate: ", type_=str) if comp_star_opt == 'y' else '', - 'y': user_input("Enter Comparison Star Y Pixel Coordinate: ", type_=str) if comp_star_opt == 'y' else '' - } + return normalized_comp_star + + for key in PHOT_COMP_STAR_KEYS: + value = comp_star.get(key, '') + if value is None: + continue + + value = str(value).strip() + normalized_comp_star[key] = '' if value.lower() in ('null', 'none') else value + + return normalized_comp_star + + +def read_aavso_metadata(prereduced_file_path): + return dict(parse_aavso_metadata(prereduced_file_path) or []) + + +def parse_aavso_metadata(prereduced_file_path): + if not prereduced_file_path: + return {} + + try: + with Path(prereduced_file_path).open('r', encoding='utf-8') as file: + for line in file: + metadata_line = line.strip() + if not metadata_line: + continue + if not metadata_line.startswith('#'): + break + if '=' not in metadata_line: + continue + + key, value = metadata_line[1:].split('=', 1) + yield key.strip().upper(), value.strip() + except (FileNotFoundError, OSError, TypeError): + return + + +def first_aavso_metadata_value(metadata, aliases): + for key in aliases: + value = metadata.get(key) + if not is_blank_value(value): + return value + return None + + +def first_aavso_metadata_text(metadata, aliases, allow_blank=False): + for key in aliases: + if key not in metadata: + continue + + value = metadata.get(key) + if value is None: + return '' if allow_blank else None + + value = str(value).strip() + if allow_blank: + return '' if value.lower() in ('null', 'none') else value + if not is_blank_value(value): + return value + return None + + +def normalize_aavso_code(value): + if value is None: + return None + value = str(value).strip() + return '' if value.lower() in ('', 'n/a', 'na', 'null', 'none') else value + + +def normalize_aavso_blankable_text(value): + if value is None: + return None + value = str(value).strip() + return '' if value.lower() in ('null', 'none') else value + + +def normalize_aavso_coordinate_text(value): + if is_blank_value(value): + return None + + value = str(value).strip() + if value[0] in ('+', '-'): + return value + + try: + numeric_value = float(value) + except ValueError: + return value + + if numeric_value >= 0: + return f"+{value}" + return value + + +def format_aavso_numeric_string(value): + value = float(value) + if value.is_integer(): + return f"{value:.1f}" + return str(value) + + +def convert_aavso_wavelength_to_nm(value, units='nm'): + if is_blank_value(value): + return None + + units_key = 'nm' if units is None else str(units).strip().lower() + units_key = units_key.replace('µ', 'u').replace('μ', 'u') + factor = AAVSO_WAVELENGTH_UNIT_FACTORS_TO_NM.get(units_key) + if factor is None: + return None + + try: + return format_aavso_numeric_string(float(str(value).strip()) * factor) + except ValueError: + return None + + +def parse_aavso_json(value): + if is_blank_value(value): + return None + + try: + return json.loads(value) + except (TypeError, json.JSONDecodeError): + return None + + +def parse_aavso_comp_star_from_metadata(metadata): + comp_star_json = metadata.get('COMP_STAR-XC') + comp_star = parse_aavso_json(comp_star_json) + if comp_star is None: + return blank_phot_comp_star() + return normalize_phot_comp_star(comp_star) + + +def lookup_aavso_filter_metadata(*candidates): + for candidate in candidates: + lookup_key = normalize_aavso_filter_lookup_key(candidate) + if lookup_key and lookup_key in AAVSO_FILTER_LOOKUP: + return AAVSO_FILTER_LOOKUP[lookup_key] + return None + + +def parse_aavso_filter_xc_fwhm(filter_metadata): + if not isinstance(filter_metadata, dict): + return None, None + + fwhm = filter_metadata.get('fwhm') + if isinstance(fwhm, dict): + values = [ + convert_aavso_wavelength_to_nm(fwhm.get('min'), fwhm.get('units', 'nm')), + convert_aavso_wavelength_to_nm(fwhm.get('max'), fwhm.get('units', 'nm')), + ] + elif isinstance(fwhm, (list, tuple)): + values = [] + for item in fwhm[:2]: + if isinstance(item, dict): + values.append(convert_aavso_wavelength_to_nm(item.get('value'), item.get('units', 'nm'))) + else: + values.append(convert_aavso_wavelength_to_nm(item)) + else: + values = [] + + values = [value for value in values if value is not None] + if len(values) < 2: + return None, None + + values = sorted(values[:2], key=float) + return values[0], values[1] + + +def parse_aavso_filter_metadata_from_metadata(metadata): + filter_value = first_aavso_metadata_text(metadata, AAVSO_FILTER_HEADER_KEYS) + filter_xc = parse_aavso_json(first_aavso_metadata_text(metadata, AAVSO_FILTER_XC_HEADER_KEYS)) + + parsed_filter = { + 'filter': filter_value, + 'filter_desc': None, + 'wl_min': None, + 'wl_max': None, + } + + if isinstance(filter_xc, dict): + filter_name = normalize_aavso_blankable_text(filter_xc.get('name')) + filter_desc = normalize_aavso_blankable_text(filter_xc.get('desc')) + if is_blank_value(parsed_filter['filter']): + parsed_filter['filter'] = filter_name or filter_desc + if filter_desc: + parsed_filter['filter_desc'] = filter_desc + + wl_min, wl_max = parse_aavso_filter_xc_fwhm(filter_xc) + if wl_min is not None and wl_max is not None: + parsed_filter['wl_min'] = wl_min + parsed_filter['wl_max'] = wl_max + + filter_record = lookup_aavso_filter_metadata( + parsed_filter['filter'], + parsed_filter['filter_desc'], + ) + if filter_record is not None: + if is_blank_value(parsed_filter['filter']): + parsed_filter['filter'] = filter_record.get('name') or filter_record.get('desc') + if is_blank_value(parsed_filter['filter_desc']): + parsed_filter['filter_desc'] = filter_record.get('desc') + if parsed_filter['wl_min'] is None: + parsed_filter['wl_min'] = filter_record['fwhm'][0] + if parsed_filter['wl_max'] is None: + parsed_filter['wl_max'] = filter_record['fwhm'][1] + + return parsed_filter + + +def parse_aavso_time_format_from_metadata(metadata): + value = first_aavso_metadata_text(metadata, AAVSO_TIME_FORMAT_HEADER_KEYS) + if is_blank_value(value): + return None + + normalized = value.upper().strip().replace('-', '_').replace(' ', '_') + if normalized in AAVSO_ALLOWED_FILE_TIME_FORMATS: + return normalized + if normalized == 'BJD': + return 'BJD_TDB' + if normalized == 'JD': + return 'JD_UTC' + if normalized == 'MJD': + return 'MJD_UTC' + return None + + +def parse_aavso_measurement_units_from_metadata(metadata): + value = first_aavso_metadata_text(metadata, AAVSO_MEASUREMENT_TYPE_HEADER_KEYS) + if is_blank_value(value): + return None + + normalized = re.sub(r'[\W_]+', '', value.lower()) + if 'millimag' in normalized or normalized == 'mmag': + return 'millimagnitude' + if 'flux' in normalized: + return 'flux' + if 'mag' in normalized: + return 'magnitude' + return None + + +def parse_aavso_exposure_from_metadata(metadata): + value = first_aavso_metadata_text(metadata, AAVSO_EXPOSURE_HEADER_KEYS) + if is_blank_value(value): + return None + + try: + return float(str(value).strip()) + except ValueError: + return None + + +def parse_aavso_airmass_detrend_from_metadata(metadata): + value = first_aavso_metadata_text(metadata, AAVSO_DETREND_PARAMETER_HEADER_KEYS, allow_blank=True) + if is_blank_value(value): + return False + + detrend_parameters = { + re.sub(r'\s+', ' ', item.strip()).upper() + for item in re.split(r'[;,]', value) + if item.strip() + } + return ( + 'AIRMASS' in detrend_parameters + and 'AIRMASS CORRECTION FUNCTION' in detrend_parameters + ) + + +def parse_aavso_prereduced_overrides(prereduced_file_path): + metadata = read_aavso_metadata(prereduced_file_path) + filter_metadata = parse_aavso_filter_metadata_from_metadata(metadata) + + return { + 'aavso_num': normalize_aavso_code(first_aavso_metadata_text(metadata, AAVSO_TEXT_HEADER_KEYS['aavso_num'], allow_blank=True)), + 'second_obs': normalize_aavso_code(first_aavso_metadata_text(metadata, AAVSO_TEXT_HEADER_KEYS['second_obs'], allow_blank=True)), + 'obs_name': normalize_aavso_blankable_text(first_aavso_metadata_text(metadata, AAVSO_TEXT_HEADER_KEYS['obs_name'], allow_blank=True)), + 'date': normalize_obs_date(first_aavso_metadata_value(metadata, AAVSO_OBSDATE_HEADER_KEYS)), + 'lat': normalize_aavso_coordinate_text(first_aavso_metadata_value(metadata, AAVSO_LOCATION_HEADER_KEYS['lat'])), + 'long': normalize_aavso_coordinate_text(first_aavso_metadata_value(metadata, AAVSO_LOCATION_HEADER_KEYS['long'])), + 'elev': first_aavso_metadata_value(metadata, AAVSO_LOCATION_HEADER_KEYS['elev']), + 'camera': first_aavso_metadata_text(metadata, AAVSO_TEXT_HEADER_KEYS['camera']), + 'pixel_bin': first_aavso_metadata_text(metadata, AAVSO_TEXT_HEADER_KEYS['pixel_bin']), + 'filter': filter_metadata['filter'], + 'filter_desc': filter_metadata['filter_desc'], + 'wl_min': filter_metadata['wl_min'], + 'wl_max': filter_metadata['wl_max'], + 'notes': normalize_aavso_blankable_text(first_aavso_metadata_text(metadata, AAVSO_TEXT_HEADER_KEYS['notes'], allow_blank=True)), + 'file_time': parse_aavso_time_format_from_metadata(metadata), + 'file_units': parse_aavso_measurement_units_from_metadata(metadata), + 'exposure': parse_aavso_exposure_from_metadata(metadata), + 'dist': first_aavso_metadata_text(metadata, AAVSO_GAIA_HEADER_KEYS['dist']), + 'pm_ra': first_aavso_metadata_text(metadata, AAVSO_GAIA_HEADER_KEYS['pm_ra']), + 'pm_dec': first_aavso_metadata_text(metadata, AAVSO_GAIA_HEADER_KEYS['pm_dec']), + 'airmass_already_corrected': parse_aavso_airmass_detrend_from_metadata(metadata), + 'phot_comp_star': parse_aavso_comp_star_from_metadata(metadata), + 'planet': first_aavso_metadata_text(metadata, AAVSO_TEXT_HEADER_KEYS['planet']), + 'host_star': first_aavso_metadata_text(metadata, AAVSO_TEXT_HEADER_KEYS['host_star']), + } + + +def parse_aavso_location(prereduced_file_path): + metadata = read_aavso_metadata(prereduced_file_path) + parsed_location = { + key: first_aavso_metadata_value(metadata, aliases) + for key, aliases in AAVSO_LOCATION_HEADER_KEYS.items() + } + parsed_location['lat'] = normalize_aavso_coordinate_text(parsed_location['lat']) + parsed_location['long'] = normalize_aavso_coordinate_text(parsed_location['long']) + return parsed_location + + +def parse_aavso_obsdate(prereduced_file_path): + metadata = read_aavso_metadata(prereduced_file_path) + return normalize_obs_date(first_aavso_metadata_value(metadata, AAVSO_OBSDATE_HEADER_KEYS)) + + +def parse_aavso_comp_star(prereduced_file_path): + metadata = read_aavso_metadata(prereduced_file_path) + return parse_aavso_comp_star_from_metadata(metadata) + + +def phot_comp_star(comp_star, prereduced_file_path=None): + if isinstance(comp_star, dict): + return normalize_phot_comp_star(comp_star) + return parse_aavso_comp_star(prereduced_file_path) + + +def first_prereduced_timestamp(prereduced_file_path): + if not prereduced_file_path: + return None + + try: + with Path(prereduced_file_path).open('r', encoding='utf-8') as file: + for line in file: + data_line = line.strip() + if not data_line or data_line.startswith('#'): + continue + + first_column = re.split(r'[\s,]+', data_line, maxsplit=1)[0] + try: + return float(first_column) + except ValueError: + continue + except (FileNotFoundError, OSError, TypeError): + return None + + return None + + +def obs_date_from_first_prereduced_entry(prereduced_file_path, time_format): + first_timestamp = first_prereduced_timestamp(prereduced_file_path) + if first_timestamp is None: + return None + + try: + if time_format == 'MJD_UTC': + return Time(first_timestamp, format='mjd', scale='utc').to_value('iso', subfmt='date') + if time_format == 'BJD_TDB': + return Time(first_timestamp, format='jd', scale='tdb').to_value('iso', subfmt='date') + if time_format == 'JD_UTC': + return Time(first_timestamp, format='jd', scale='utc').to_value('iso', subfmt='date') + except (TypeError, ValueError): + return None + + return None + + +def prereduced_obs_date(date, prereduced_file_path=None, time_format=None): + aavso_obsdate = parse_aavso_obsdate(prereduced_file_path) + if aavso_obsdate is not None: + return aavso_obsdate + + derived_obsdate = obs_date_from_first_prereduced_entry(prereduced_file_path, time_format) + if derived_obsdate is not None: + return derived_obsdate + + normalized_date = normalize_obs_date(date) + if normalized_date is not None: + return normalized_date - return comp_star + return "" def data_file_time(time_format): @@ -620,7 +1742,7 @@ def log_info(string, warn=False, error=False): if error: print(f"\033[91m {string}\033[00m") elif warn: - print(f"\033[93m {string}\033[00m") + print(f"\033[34m {string}\033[00m") else: print(string) log.debug(string) diff --git a/exotic/output_files.py b/exotic/output_files.py index 87e97681..439d5ab4 100644 --- a/exotic/output_files.py +++ b/exotic/output_files.py @@ -1,11 +1,45 @@ from json import dump, dumps -from numpy import mean, median, std +import shutil +from numpy import mean, std from pathlib import Path +import numpy as np try: - from utils import round_to_2 + from utils import ( + aavso_output_directory, + MAGNITUDE_DECIMAL_PLACES, + filename_date_token, + format_aavso_exoplanet_name, + format_magnitude_error, + format_magnitude, + magnitude_text, + format_uncertainty, + format_value_and_uncertainty, + format_value_with_uncertainty, + normalized_magnitude_error, + round_to_2, + rounded_magnitude_error, + rounded_magnitude_value, + safe_output_filename, + ) except ImportError: - from .utils import round_to_2 + from .utils import ( + aavso_output_directory, + MAGNITUDE_DECIMAL_PLACES, + filename_date_token, + format_aavso_exoplanet_name, + format_magnitude_error, + format_magnitude, + magnitude_text, + format_uncertainty, + format_value_and_uncertainty, + format_value_with_uncertainty, + normalized_magnitude_error, + round_to_2, + rounded_magnitude_error, + rounded_magnitude_value, + safe_output_filename, + ) try: from version import __version__ except ImportError: @@ -14,6 +48,2477 @@ from plate_status import PlateStatus except ImportError: from .plate_status import PlateStatus +try: + from transit_depth import ( + AREA_DEPTH_LABEL, + OBSERVABLE_DEPTH_DELTA_LABEL, + OBSERVABLE_DEPTH_LABEL, + PRIOR_OBSERVABLE_DEPTH_LABEL, + fit_transit_depth_summary, + planet_dict_transit_errors, + planet_dict_transit_parameters, + ) +except ImportError: + from .transit_depth import ( + AREA_DEPTH_LABEL, + OBSERVABLE_DEPTH_DELTA_LABEL, + OBSERVABLE_DEPTH_LABEL, + PRIOR_OBSERVABLE_DEPTH_LABEL, + fit_transit_depth_summary, + planet_dict_transit_errors, + planet_dict_transit_parameters, + ) + + +AAVSO_FINDER_STRETCH_NAMES = ( + 'LinearStretch', + 'SquaredStretch', + 'SqrtStretch', + 'LogStretch', +) + + +def copy_aavso_supporting_artifacts(save, target_name, observation_date): + """Copy final lightcurve, finder, triangle, and QC products into ``AAVSO_Files``.""" + + output_dir = Path(save) + working_artifacts_dir = output_dir / 'working_artifacts' + diagnostics_dir = output_dir / 'Diagnostics' + date_token = filename_date_token(observation_date) + source_paths = [ + output_dir / safe_output_filename( + 'FinalLightCurve', target_name, date_token, extension=extension + ) + for extension in ('png', 'pdf') + ] + source_paths.append( + working_artifacts_dir / safe_output_filename( + 'FinalLightCurve', target_name, date_token, extension='csv' + ) + ) + for stretch_name in AAVSO_FINDER_STRETCH_NAMES: + source_paths.extend( + working_artifacts_dir / safe_output_filename( + 'FOV', target_name, stretch_name, date_token, extension=extension + ) + for extension in ('png', 'pdf') + ) + for prefix, extensions in ( + ('FinalTriangle', ('png',)), + ('Triangle', ('png',)), + ('ZoomedTrianglePlot', ('png',)), + ('KTMF_QC', ('png', 'pdf')), + ('PriorPosteriorComparison', ('png', 'pdf')), + ): + source_paths.extend( + diagnostics_dir / safe_output_filename( + prefix, target_name, date_token, extension=extension + ) + for extension in extensions + ) + + aavso_dir = aavso_output_directory(output_dir) + copied_paths = [] + for source_path in source_paths: + if not source_path.is_file(): + continue + destination_path = aavso_dir / source_path.name + shutil.copy2(source_path, destination_path) + copied_paths.append(destination_path) + return copied_paths + + +def aavso_airmass_results(fit): + if getattr(fit, 'airmass_fit_skipped', False): + return ( + ('Am1', '0', '0'), + ('Am2', '0', '0'), + ) + + if baseline_fixed_after_detrending(fit): + first_key = 'A0' if 'a0' in fit.parameters else 'Am1' + first_value = fit.parameters.get('a0', fit.parameters.get('a1', 1.0)) + return ( + (first_key, str(round_to_2(first_value)), '0'), + ('Am2', str(round_to_2(fit.parameters.get('a2', 0.0))), '0'), + ) + + if 'a0' in fit.parameters: + value_text, uncertainty_text = format_value_and_uncertainty( + fit.parameters['a0'], fit.errors['a0'] + ) + first_result = ( + 'A0', + value_text, + uncertainty_text, + ) + else: + value_text, uncertainty_text = format_value_and_uncertainty( + fit.parameters['a1'], fit.errors['a1'] + ) + first_result = ( + 'Am1', + value_text, + uncertainty_text, + ) + + a2_value_text, a2_uncertainty_text = format_value_and_uncertainty( + fit.parameters.get('a2', 0), fit.errors.get('a2', 0) + ) + + return ( + first_result, + ( + 'Am2', + a2_value_text, + a2_uncertainty_text, + ), + ) + + +def aavso_detrend_model(fit): + oot_baseline_model = out_of_transit_baseline_model(fit) + if oot_baseline_model is not None: + return oot_baseline_model + if getattr(fit, 'airmass_fit_skipped', False): + return np.ones(len(fit.time), dtype=float) + return np.asarray(fit.airmass_model, dtype=float) + + +def out_of_transit_baseline_model(fit): + """Rebuild the linear baseline divided out before the final transit fit.""" + if not baseline_fixed_after_detrending(fit): + return None + + times = np.asarray(getattr(fit, 'time', []), dtype=float).reshape(-1) + slope = finite_float(getattr(fit, 'oot_baseline_slope', None)) + intercept = finite_float(getattr(fit, 'oot_baseline_intercept', None)) + reference_time = finite_float( + getattr(fit, 'oot_baseline_reference_time_bjd_tdb', None) + ) + if ( + times.size == 0 + or not np.isfinite(slope) + or not np.isfinite(intercept) + or not np.isfinite(reference_time) + ): + return None + + baseline = intercept + slope * (times - reference_time) + if baseline.shape != times.shape or not np.all(np.isfinite(baseline)) or np.any(baseline <= 0): + return None + return baseline + + +def out_of_transit_baseline_detrending_metadata(fit): + """Return the complete reversible linear-baseline contract for AAVSO output.""" + applied = baseline_fixed_after_detrending(fit) + metadata = { + 'applied': applied, + 'note': getattr(fit, 'oot_baseline_detrending_note', None), + } + if not applied: + return metadata + + metadata.update({ + 'model': 'baseline(t) = intercept + slope_per_day * (BJD_TDB - reference_time_bjd_tdb)', + 'forward_correction': 'detrended_flux = raw_flux / baseline(t)', + 'inverse_correction': 'raw_flux = detrended_flux * baseline(t)', + 'reference_time_bjd_tdb': finite_float( + getattr(fit, 'oot_baseline_reference_time_bjd_tdb', None) + ), + 'intercept': finite_float(getattr(fit, 'oot_baseline_intercept', None)), + 'slope_per_day': finite_float(getattr(fit, 'oot_baseline_slope', None)), + 'pre_ingress_point_count': int(getattr(fit, 'oot_baseline_pre_points', 0) or 0), + 'post_egress_point_count': int(getattr(fit, 'oot_baseline_post_points', 0) or 0), + 'serialized_model_available': out_of_transit_baseline_model(fit) is not None, + }) + return metadata + + +def aavso_undetrended_flux_series(fit, detrend_model): + """Return pre-correction flux/error arrays matching the exported correction model.""" + data = np.asarray(getattr(fit, 'data', []), dtype=float).reshape(-1) + data_error = np.asarray(getattr(fit, 'dataerr', []), dtype=float).reshape(-1) + detrend_model = np.asarray(detrend_model, dtype=float).reshape(-1) + if not (data.shape == data_error.shape == detrend_model.shape): + return data, data_error + oot_baseline_model = out_of_transit_baseline_model(fit) + if oot_baseline_model is not None and oot_baseline_model.shape == data.shape: + return data * oot_baseline_model, data_error * oot_baseline_model + return data, data_error + + +def baseline_fixed_after_detrending(fit): + """Return whether the reported neutral baseline was fixed after detrending.""" + + return bool(getattr(fit, 'oot_baseline_detrending_applied', False)) + + +def fixed_detrended_baseline_text(value): + return f"{round_to_2(value)} (fixed after out-of-transit baseline detrending)" + + +def pre_detrending_baseline_report(fit): + """Return formatted measured baseline coefficients retained before detrending.""" + scale_parameter = getattr(fit, 'pre_detrending_baseline_scale_parameter', None) + scale_value = getattr(fit, 'pre_detrending_baseline_scale_value', None) + scale_error = getattr(fit, 'pre_detrending_baseline_scale_error', None) + a2_value = getattr(fit, 'pre_detrending_baseline_a2_value', None) + a2_error = getattr(fit, 'pre_detrending_baseline_a2_error', None) + + def formatted_measurement(value, error): + try: + value_is_finite = np.isfinite(value) + except TypeError: + value_is_finite = False + if not value_is_finite: + return None + try: + error_is_finite = np.isfinite(error) and float(error) >= 0 + except (TypeError, ValueError): + error_is_finite = False + if error_is_finite: + return format_value_with_uncertainty(value, error) + return f"{round_to_2(value)} (uncertainty unavailable)" + + scale_text = formatted_measurement(scale_value, scale_error) + a2_text = formatted_measurement(a2_value, a2_error) + if scale_text is None and a2_text is None: + return None + + return { + 'source': getattr(fit, 'pre_detrending_baseline_source', None), + 'scale_parameter': scale_parameter if scale_parameter in {'a0', 'a1'} else 'a1', + 'scale_text': scale_text, + 'a2_text': a2_text, + } + + +def finite_float(value, default=np.nan): + try: + value = float(value) + except (TypeError, ValueError): + return default + return value if np.isfinite(value) else default + + +def apparent_magnitude_calibration_from_vsp_params(vsp_params): + rows = [] + zero_points = [] + zero_point_errors = [] + for vsp_p in vsp_params or []: + mag = finite_float(vsp_p.get('mag')) + mag_err = normalized_magnitude_error(vsp_p.get('mag_err')) + if not np.isfinite(mag) or mag_err is None: + continue + rows.append((mag, mag_err, vsp_p.get('mag_band') or 'V')) + + differential_mag, differential_err = differential_magnitude_from_vsp_param( + vsp_p + ) + if np.isfinite(differential_mag): + comparison_mag = finite_float(vsp_p.get('cmag')) + comparison_mag_err = normalized_magnitude_error(vsp_p.get('cmag_err')) + zero_points.append( + comparison_mag if np.isfinite(comparison_mag) else mag - differential_mag + ) + if comparison_mag_err is not None: + zero_point_errors.append(comparison_mag_err) + elif np.isfinite(differential_err): + zero_point_errors.append( + np.sqrt(max(mag_err ** 2 - differential_err ** 2, 0.0)) + ) + else: + zero_point_errors.append(mag_err) + continue + + comparison_mag = finite_float(vsp_p.get('cmag')) + comparison_mag_err = normalized_magnitude_error(vsp_p.get('cmag_err')) + if np.isfinite(comparison_mag): + zero_points.append(comparison_mag) + zero_point_errors.append( + comparison_mag_err if comparison_mag_err is not None else mag_err + ) + + if not rows: + return None + + magnitudes = np.array([row[0] for row in rows], dtype=float) + magnitude_errors = np.array([row[1] for row in rows], dtype=float) + return { + 'baseline_magnitude': float(np.nanmedian(magnitudes)), + 'baseline_error': float(np.nanmedian(magnitude_errors)), + 'band': rows[0][2], + 'zero_point_magnitude': ( + float(np.nanmedian(zero_points)) if zero_points else np.nan + ), + 'zero_point_error': ( + float(np.nanmedian(zero_point_errors)) if zero_point_errors else np.nan + ), + } + + +def differential_magnitude_from_vsp_param(vsp_param): + """Return target-minus-reference magnitude and its flux-only uncertainty.""" + differential_mag = finite_float( + vsp_param.get( + 'differential_mag', + vsp_param.get('differential_magnitude'), + ) + ) + differential_err = finite_float( + vsp_param.get( + 'differential_mag_err', + vsp_param.get('differential_magnitude_error'), + ) + ) + if np.isfinite(differential_err) and differential_err < 0: + differential_err = np.nan + + if np.isfinite(differential_mag): + return differential_mag, differential_err + + apparent_mag = finite_float(vsp_param.get('mag')) + comparison_mag = finite_float(vsp_param.get('cmag')) + if not (np.isfinite(apparent_mag) and np.isfinite(comparison_mag)): + return np.nan, np.nan + + apparent_err = normalized_magnitude_error(vsp_param.get('mag_err')) + comparison_err = normalized_magnitude_error(vsp_param.get('cmag_err')) + if apparent_err is not None and comparison_err is not None: + differential_err = np.sqrt(max(apparent_err ** 2 - comparison_err ** 2, 0.0)) + elif apparent_err is not None: + differential_err = apparent_err + else: + differential_err = np.nan + return apparent_mag - comparison_mag, differential_err + + +def differential_magnitude_correction_model(fit, shape): + """Return the relative correction applied to raw target/reference ratios.""" + shape = tuple(shape) + correction_type = 'none' + correction_model = out_of_transit_baseline_model(fit) + if correction_model is not None: + correction_type = 'out_of_transit_linear_baseline' + elif baseline_fixed_after_detrending(fit): + correction_type = 'unavailable' + elif not getattr(fit, 'airmass_fit_skipped', False): + candidate = getattr(fit, 'airmass_model', None) + if candidate is not None: + candidate = np.asarray(candidate, dtype=float).reshape(-1) + if candidate.shape == shape: + correction_model = candidate + correction_type = 'airmass' + + if correction_model is None or np.asarray(correction_model).shape != shape: + correction_model = np.ones(shape, dtype=float) + if correction_type != 'unavailable': + correction_type = 'none' + else: + correction_model = np.asarray(correction_model, dtype=float).reshape(-1) + + valid = np.isfinite(correction_model) & (correction_model > 0) + reference = float(np.nanmedian(correction_model[valid])) if np.any(valid) else 1.0 + if not np.isfinite(reference) or reference <= 0: + reference = 1.0 + relative_model = np.divide( + correction_model, + reference, + out=np.full(shape, np.nan, dtype=float), + where=valid, + ) + correction_applied = bool( + correction_type != 'none' + and np.any(valid) + and not np.allclose(relative_model[valid], 1.0, rtol=0.0, atol=1.0e-12) + ) + return { + 'model': correction_model, + 'relative_model': relative_model, + 'reference': reference, + 'type': correction_type, + 'applied': correction_applied, + } + + +def differential_magnitude_series_from_fit(fit, out_of_transit_only=False, + apply_airmass_correction=None): + """Return target-minus-reference instrumental magnitudes. + + Unlike apparent magnitudes, this series needs no catalogue magnitude. Raw + target/reference fluxes are preferred so the instrumental zero point is + preserved; pre-reduced light curves fall back to their relative flux. + Stellar-variability fits intentionally remain uncorrected for airmass so a + real time-dependent stellar signal is not fitted away. + """ + fit_data = np.asarray( + getattr(fit, 'data', getattr(fit, 'detrended', [])), + dtype=float, + ).reshape(-1) + fit_times = np.asarray( + getattr(fit, 'time', getattr(fit, 'jd_times', [])), + dtype=float, + ).reshape(-1) + if fit_data.size == 0 or fit_times.shape != fit_data.shape: + return None + + if apply_airmass_correction is None: + apply_airmass_correction = not bool( + getattr(fit, 'stellar_variability_only', False) + ) + apply_airmass_correction = bool(apply_airmass_correction) + + target_flux = np.asarray( + getattr( + fit, + 'differential_magnitude_target_flux', + getattr(fit, 'stellar_variability_target_flux', []), + ), + dtype=float, + ).reshape(-1) + reference_flux = np.asarray( + getattr( + fit, + 'differential_magnitude_reference_flux', + getattr(fit, 'stellar_variability_comp_flux', []), + ), + dtype=float, + ).reshape(-1) + target_error = np.asarray( + getattr( + fit, + 'differential_magnitude_target_flux_error', + getattr(fit, 'stellar_variability_target_flux_error', []), + ), + dtype=float, + ).reshape(-1) + reference_error = np.asarray( + getattr( + fit, + 'differential_magnitude_reference_flux_error', + getattr(fit, 'stellar_variability_comp_flux_error', []), + ), + dtype=float, + ).reshape(-1) + + has_raw_photometry = ( + target_flux.shape == fit_data.shape + and reference_flux.shape == fit_data.shape + ) + correction = differential_magnitude_correction_model(fit, fit_data.shape) + relative_correction_model = correction['relative_model'] + if not has_raw_photometry: + # Ordinary fits retain their uncorrected relative flux in ``fit.data``. + # A linear out-of-transit pass instead leaves corrected data on the + # final fit, so restore its raw input with the retained baseline model. + target_flux = fit_data.copy() + target_error = np.asarray(getattr(fit, 'dataerr', []), dtype=float).reshape(-1) + if baseline_fixed_after_detrending(fit) and correction['type'] == 'out_of_transit_linear_baseline': + target_flux = target_flux * relative_correction_model + if target_error.shape == fit_data.shape: + target_error = target_error * relative_correction_model + reference_flux = np.ones(fit_data.shape, dtype=float) + reference_error = np.zeros(fit_data.shape, dtype=float) + + if target_error.shape != fit_data.shape: + target_error = np.full(fit_data.shape, np.nan, dtype=float) + if reference_error.shape != fit_data.shape: + reference_error = np.full(fit_data.shape, np.nan, dtype=float) + + with np.errstate(divide='ignore', invalid='ignore'): + raw_ratio = np.divide(target_flux, reference_flux) + corrected_ratio = ( + np.divide(raw_ratio, relative_correction_model) + if apply_airmass_correction + else raw_ratio + ) + differential_magnitude = -2.5 * np.log10(corrected_ratio) + magnitude_factor = 2.5 / np.log(10.0) + explicit_error = magnitude_factor * np.sqrt( + (target_error / target_flux) ** 2 + + (reference_error / reference_flux) ** 2 + ) + + fit_error = np.asarray(getattr(fit, 'dataerr', []), dtype=float).reshape(-1) + if fit_error.shape == fit_data.shape: + with np.errstate(divide='ignore', invalid='ignore'): + fallback_error = magnitude_factor * np.abs(fit_error / fit_data) + else: + fallback_error = np.full(fit_data.shape, np.nan, dtype=float) + differential_error = np.where( + np.isfinite(explicit_error) & (explicit_error >= 0), + explicit_error, + fallback_error, + ) + + airmass = np.asarray( + getattr(fit, 'airmass', np.full(fit_data.shape, np.nan)), + dtype=float, + ).reshape(-1) + if airmass.shape != fit_data.shape: + airmass = np.full(fit_data.shape, np.nan, dtype=float) + + keep = ( + np.isfinite(fit_times) + & np.isfinite(corrected_ratio) + & (corrected_ratio > 0) + & np.isfinite(differential_magnitude) + ) + if out_of_transit_only: + transit_model = np.asarray( + getattr(fit, 'transit', np.ones(fit_data.shape)), + dtype=float, + ).reshape(-1) + if transit_model.shape == fit_data.shape and np.any(transit_model == 1): + keep &= transit_model == 1 + + if not np.any(keep): + return None + return { + 'time': fit_times[keep], + 'airmass': airmass[keep], + 'magnitude': differential_magnitude[keep], + 'magnitude_error': differential_error[keep], + 'source_mask': keep, + 'airmass_corrected': bool( + apply_airmass_correction + and correction['type'] == 'airmass' + and correction['applied'] + ), + 'correction_applied': bool(apply_airmass_correction and correction['applied']), + 'correction_type': correction['type'] if apply_airmass_correction else 'none', + 'correction_factor': ( + relative_correction_model[keep] + if apply_airmass_correction + else np.ones(np.count_nonzero(keep), dtype=float) + ), + 'raw_measurement_available': bool( + has_raw_photometry + or not ( + correction['type'] == 'unavailable' + or ( + getattr(fit, 'airmass_fit_skipped', False) + and 'input AAVSO file already reports' in str( + getattr(fit, 'airmass_correction_note', '') + ) + ) + ) + ), + 'has_raw_photometry': has_raw_photometry, + } + + +def magnitude_series_from_fit(fit, out_of_transit_only=False, + apply_airmass_correction=None): + """Return aligned differential and, when calibrated, apparent magnitudes.""" + fit_data = np.asarray( + getattr(fit, 'data', getattr(fit, 'detrended', [])), + dtype=float, + ).reshape(-1) + result = { + 'differential_magnitude': np.full(fit_data.shape, np.nan, dtype=float), + 'differential_magnitude_error': np.full(fit_data.shape, np.nan, dtype=float), + 'raw_differential_magnitude': np.full(fit_data.shape, np.nan, dtype=float), + 'raw_differential_magnitude_error': np.full(fit_data.shape, np.nan, dtype=float), + 'corrected_differential_magnitude': np.full(fit_data.shape, np.nan, dtype=float), + 'corrected_differential_magnitude_error': np.full(fit_data.shape, np.nan, dtype=float), + 'differential_magnitude_correction_factor': np.full(fit_data.shape, np.nan, dtype=float), + 'apparent_magnitude': np.full(fit_data.shape, np.nan, dtype=float), + 'apparent_magnitude_error': np.full(fit_data.shape, np.nan, dtype=float), + 'band': None, + 'airmass_corrected': False, + 'correction_applied': False, + 'correction_type': 'none', + 'raw_measurement_available': False, + 'has_raw_photometry': False, + 'apparent_calibrated': False, + } + series = differential_magnitude_series_from_fit( + fit, + out_of_transit_only=out_of_transit_only, + apply_airmass_correction=apply_airmass_correction, + ) + if series is None: + return result + + source_mask = np.asarray(series['source_mask'], dtype=bool) + if source_mask.shape != fit_data.shape: + return result + result['differential_magnitude'][source_mask] = series['magnitude'] + result['differential_magnitude_error'][source_mask] = series['magnitude_error'] + result['airmass_corrected'] = bool(series['airmass_corrected']) + result['correction_applied'] = bool(series['correction_applied']) + result['correction_type'] = series['correction_type'] + result['raw_measurement_available'] = bool(series['raw_measurement_available']) + result['has_raw_photometry'] = bool(series['has_raw_photometry']) + + raw_series = differential_magnitude_series_from_fit( + fit, + out_of_transit_only=out_of_transit_only, + apply_airmass_correction=False, + ) + if raw_series is not None and raw_series.get('raw_measurement_available', False): + raw_mask = np.asarray(raw_series['source_mask'], dtype=bool) + if raw_mask.shape == fit_data.shape: + result['raw_differential_magnitude'][raw_mask] = raw_series['magnitude'] + result['raw_differential_magnitude_error'][raw_mask] = raw_series['magnitude_error'] + + correct_variability = ( + not bool(getattr(fit, 'stellar_variability_only', False)) + if apply_airmass_correction is None + else bool(apply_airmass_correction) + ) + corrected_series = differential_magnitude_series_from_fit( + fit, + out_of_transit_only=out_of_transit_only, + apply_airmass_correction=correct_variability, + ) + if corrected_series is not None: + corrected_mask = np.asarray(corrected_series['source_mask'], dtype=bool) + if corrected_mask.shape == fit_data.shape: + result['corrected_differential_magnitude'][corrected_mask] = corrected_series['magnitude'] + result['corrected_differential_magnitude_error'][corrected_mask] = ( + corrected_series['magnitude_error'] + ) + result['differential_magnitude_correction_factor'][corrected_mask] = ( + corrected_series['correction_factor'] + ) + + calibration = apparent_magnitude_calibration_from_vsp_params( + getattr(fit, 'stellar_variability_params', None) + ) + if calibration is None: + return result + + # A calibrated comparison ensemble already carries its independently + # derived apparent-magnitude series. Do not reconstruct that series by + # adding a constant to the raw instrumental differential magnitudes: the + # calibrated ensemble and the raw median-scaled ensemble intentionally use + # different reference constructions and can have different time trends. + calibrated_magnitude = np.asarray( + getattr(fit, 'stellar_variability_ensemble_magnitudes', []), + dtype=float, + ).reshape(-1) + calibrated_error = np.asarray( + getattr(fit, 'stellar_variability_ensemble_magnitude_errors', []), + dtype=float, + ).reshape(-1) + if calibrated_magnitude.shape == fit_data.shape: + calibrated_mask = source_mask & np.isfinite(calibrated_magnitude) + result['apparent_magnitude'][calibrated_mask] = calibrated_magnitude[calibrated_mask] + if calibrated_error.shape == fit_data.shape: + calibrated_error_mask = calibrated_mask & np.isfinite(calibrated_error) + result['apparent_magnitude_error'][calibrated_error_mask] = ( + calibrated_error[calibrated_error_mask] + ) + result['band'] = calibration['band'] + result['apparent_calibrated'] = bool(np.any(calibrated_mask)) + return result + + magnitude_offset = np.nan + calibration_error = calibration['baseline_error'] + if np.isfinite(calibration['zero_point_error']): + calibration_error = calibration['zero_point_error'] + + differential_magnitude = result['differential_magnitude'] + differential_error = result['differential_magnitude_error'] + fit_times = np.asarray( + getattr(fit, 'time', getattr(fit, 'jd_times', [])), + dtype=float, + ).reshape(-1) + matched_offsets = [] + if result['has_raw_photometry'] and fit_times.shape == differential_magnitude.shape: + for vsp_param in getattr(fit, 'stellar_variability_params', None) or []: + apparent_mag = finite_float(vsp_param.get('mag')) + apparent_time = finite_float(vsp_param.get('time')) + if not (np.isfinite(apparent_mag) and np.isfinite(apparent_time)): + continue + matches = np.flatnonzero(np.isclose( + fit_times, + apparent_time, + rtol=0.0, + atol=1.0e-7, + )) + if matches.size == 0: + continue + matched_differential = differential_magnitude[matches[0]] + if np.isfinite(matched_differential): + matched_offsets.append(apparent_mag - matched_differential) + if not result['has_raw_photometry']: + magnitude_offset = calibration['baseline_magnitude'] + elif matched_offsets: + magnitude_offset = float(np.nanmedian(matched_offsets)) + else: + reference_mask = np.isfinite(differential_magnitude) + transit_model = np.asarray( + getattr(fit, 'transit', np.ones(fit_data.shape)), + dtype=float, + ).reshape(-1) + if transit_model.shape == fit_data.shape and np.any(transit_model == 1): + reference_mask &= transit_model == 1 + if np.any(reference_mask): + magnitude_offset = ( + calibration['baseline_magnitude'] + - float(np.nanmedian(differential_magnitude[reference_mask])) + ) + if not np.isfinite(magnitude_offset): + return result + + finite_differential = np.isfinite(differential_magnitude) + result['apparent_magnitude'][finite_differential] = ( + differential_magnitude[finite_differential] + magnitude_offset + ) + if np.isfinite(calibration_error): + finite_error = finite_differential & np.isfinite(differential_error) + result['apparent_magnitude_error'][finite_error] = np.hypot( + differential_error[finite_error], + calibration_error, + ) + missing_error = finite_differential & ~np.isfinite(differential_error) + result['apparent_magnitude_error'][missing_error] = calibration_error + else: + result['apparent_magnitude_error'][finite_differential] = ( + differential_error[finite_differential] + ) + result['band'] = calibration['band'] + result['apparent_calibrated'] = True + return result + + +def write_differential_magnitude_csv(fit, save, target_name, observation_date=None, + observed_filter=None, out_of_transit_only=False, + apply_airmass_correction=None, + filename_prefix='DifferentialMagnitude'): + series = magnitude_series_from_fit( + fit, + out_of_transit_only=out_of_transit_only, + apply_airmass_correction=apply_airmass_correction, + ) + times = np.asarray( + getattr(fit, 'time', getattr(fit, 'jd_times', [])), + dtype=float, + ).reshape(-1) + airmass = np.asarray( + getattr(fit, 'airmass', np.full(times.shape, np.nan)), + dtype=float, + ).reshape(-1) + if airmass.shape != times.shape: + airmass = np.full(times.shape, np.nan, dtype=float) + raw_magnitude = np.asarray(series['raw_differential_magnitude'], dtype=float) + corrected_magnitude = np.asarray(series['corrected_differential_magnitude'], dtype=float) + if not ( + raw_magnitude.shape == corrected_magnitude.shape == times.shape + and np.any(np.isfinite(raw_magnitude) | np.isfinite(corrected_magnitude)) + ): + return None + + output_dir = Path(save) + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / safe_output_filename( + filename_prefix, + target_name, + filename_date_token(observation_date) if observation_date else 'undated', + extension='csv', + ) + comparison = ( + getattr(fit, 'stellar_variability_reference_label', None) + or getattr(fit, 'differential_magnitude_reference_label', None) + or 'selected comparison reference' + ) + with output_path.open('w', encoding='utf-8') as handle: + handle.write( + '# AIRMASS_CORRECTION=' + f"{'YES' if series['airmass_corrected'] else 'NO'}\n" + ) + handle.write(f"# DIFFERENTIAL_MAGNITUDE_CORRECTION={series['correction_type']}\n") + handle.write( + '# BJD_TDB,Airmass,Raw Differential Magnitude,' + 'Raw Differential Magnitude Uncertainty,Corrected Differential Magnitude,' + 'Corrected Differential Magnitude Uncertainty,Correction Factor,Filter,Comparison\n' + ) + for index, time_value in enumerate(times): + if not ( + np.isfinite(raw_magnitude[index]) + or np.isfinite(corrected_magnitude[index]) + ): + continue + airmass_text = f"{airmass[index]}" if np.isfinite(airmass[index]) else 'na' + raw_text = format_magnitude( + raw_magnitude[index], default='na', digits=MAGNITUDE_DECIMAL_PLACES + ) + raw_error_text = format_magnitude_error( + series['raw_differential_magnitude_error'][index], + default='na', + digits=MAGNITUDE_DECIMAL_PLACES, + ) + corrected_text = format_magnitude( + corrected_magnitude[index], default='na', digits=MAGNITUDE_DECIMAL_PLACES + ) + corrected_error_text = format_magnitude_error( + series['corrected_differential_magnitude_error'][index], + default='na', + digits=MAGNITUDE_DECIMAL_PLACES, + ) + correction_factor = finite_float( + series['differential_magnitude_correction_factor'][index] + ) + correction_factor_text = ( + f"{correction_factor:.7f}" if np.isfinite(correction_factor) else 'na' + ) + handle.write( + f"{time_value}, {airmass_text}, " + f"{raw_text}, {raw_error_text}, {corrected_text}, {corrected_error_text}, " + f"{correction_factor_text}, " + f"{observed_filter or 'na'}, {comparison}\n" + ) + return output_path + + +def aavso_json_safe(value): + if isinstance(value, dict): + return {str(key): aavso_json_safe(subvalue) for key, subvalue in value.items()} + if isinstance(value, (list, tuple)): + return [aavso_json_safe(item) for item in value] + if isinstance(value, np.ndarray): + if value.ndim == 0: + return aavso_json_safe(value.item()) + return [aavso_json_safe(item) for item in value.tolist()] + if isinstance(value, np.generic): + return aavso_json_safe(value.item()) + if isinstance(value, Path): + return str(value) + if isinstance(value, bool): + return bool(value) + if isinstance(value, float): + return float(value) if np.isfinite(value) else None + if isinstance(value, int): + return int(value) + return value + + +def format_optional_float(value, digits=7): + value = finite_float(value) + if np.isfinite(value): + return f"{value:.{digits}f}" + return "na" + + +def stellar_variability_reference_summary(vsp_param): + if not vsp_param: + return "na" + + cname = vsp_param.get('cname', 'na') + band = vsp_param.get('catalog_mag_band') or vsp_param.get('mag_band') or 'V' + if vsp_param.get('ensemble_reference'): + member_count = int(vsp_param.get('ensemble_member_count', 0) or 0) + labels = vsp_param.get('ensemble_member_labels') or [] + label_text = ", ".join(str(label) for label in labels) + return ( + f"Calibrated comparison-star ensemble ({member_count} stars)" + + (f": {label_text}" if label_text else "") + ) + if vsp_param.get('is_aavso_vsp', True): + return f"AAVSO Label: {cname}, Position: {vsp_param.get('pos')}" + + source = vsp_param.get('catalog_source') or 'NextAstro photometry catalog' + comp_ra = format_optional_float(vsp_param.get('comp_ra')) + comp_dec = format_optional_float(vsp_param.get('comp_dec')) + details = [f"{source}: RA={comp_ra}", f"Dec={comp_dec}"] + mag_text = magnitude_text(band, vsp_param.get('cmag'), vsp_param.get('cmag_err')) + if mag_text is not None: + details.append(mag_text) + return ", ".join(details) + + +def stellar_variability_measurement_summary(vsp_params, transit_fit_comp_star=None): + point_count = len(vsp_params or []) + if point_count == 0: + return None + + if transit_fit_comp_star is None: + return None + + if vsp_params[0].get('ensemble_reference'): + return ( + f"Combined {point_count} out-of-transit target measurements against the calibrated " + "comparison-star ensemble; AID rows list the BJD_TDB timestamps used." + ) + + return ( + f"Remeasured {point_count} out-of-transit target/reference point(s) against the transit-fit " + f"{'derived ' if vsp_params[0].get('derived_catalog_reference') else ''}catalog reference " + "for AID magnitudes; AID rows list the BJD_TDB timestamps used." + ) + + +def aid_comparison_metadata(vsp_param): + if not vsp_param: + return {} + anchor_labels = vsp_param.get('derived_reference_anchor_labels') + anchor_label_sample = None + if isinstance(anchor_labels, np.ndarray): + anchor_labels = anchor_labels.tolist() + if isinstance(anchor_labels, (list, tuple)): + anchor_labels = list(anchor_labels) + if len(anchor_labels) > 10: + anchor_label_sample = anchor_labels[:10] + anchor_labels = None + metadata = { + 'source': vsp_param.get('catalog_source', 'AAVSO VSP'), + 'is_aavso_vsp': bool(vsp_param.get('is_aavso_vsp', True)), + 'comparison_name': vsp_param.get('cname'), + 'comparison_position_pixels': vsp_param.get('pos'), + 'comparison_ra_deg': vsp_param.get('comp_ra'), + 'comparison_dec_deg': vsp_param.get('comp_dec'), + 'catalog_ra_deg': vsp_param.get('catalog_ra'), + 'catalog_dec_deg': vsp_param.get('catalog_dec'), + 'catalog_source_id': vsp_param.get('source_id'), + 'catalog_id': vsp_param.get('catalog_id'), + 'catalog_match_separation_arcsec': vsp_param.get('separation_arcsec'), + 'derived_catalog_reference': bool(vsp_param.get('derived_catalog_reference', False)), + 'derived_reference_anchor_count': vsp_param.get('derived_reference_anchor_count'), + 'magnitude_band': vsp_param.get('catalog_mag_band') or vsp_param.get('mag_band'), + 'reported_measurement_band': vsp_param.get('mag_band'), + 'apparent_magnitude': rounded_magnitude_value(vsp_param.get('cmag')), + 'apparent_magnitude_error': rounded_magnitude_error(vsp_param.get('cmag_err')), + 'ensemble_reference': bool(vsp_param.get('ensemble_reference', False)), + 'ensemble_member_count': vsp_param.get('ensemble_member_count'), + 'ensemble_member_labels': vsp_param.get('ensemble_member_labels'), + 'ensemble_member_positions': vsp_param.get('ensemble_member_positions'), + 'ensemble_member_catalog_magnitudes': vsp_param.get('ensemble_member_catalog_magnitudes'), + 'ensemble_member_catalog_errors': vsp_param.get('ensemble_member_catalog_errors'), + 'ensemble_member_catalog_sources': vsp_param.get('ensemble_member_catalog_sources'), + 'ensemble_member_ra_degs': vsp_param.get('ensemble_member_ra_degs'), + 'ensemble_member_dec_degs': vsp_param.get('ensemble_member_dec_degs'), + 'ensemble_members': vsp_param.get('ensemble_members'), + 'ensemble_member_catalog_colors': vsp_param.get('ensemble_member_catalog_colors'), + 'ensemble_member_catalog_color_labels': vsp_param.get('ensemble_member_catalog_color_labels'), + 'ensemble_member_color_deltas': vsp_param.get('ensemble_member_color_deltas'), + 'ensemble_member_magnitude_deltas': vsp_param.get('ensemble_member_magnitude_deltas'), + 'ensemble_member_similarity_scores': vsp_param.get('ensemble_member_similarity_scores'), + } + if anchor_labels is not None: + metadata['derived_reference_anchor_labels'] = anchor_labels + if anchor_label_sample is not None: + metadata['derived_reference_anchor_label_sample'] = anchor_label_sample + return aavso_json_safe(metadata) + + +def aid_comparison_coordinate_headers(vsp_params, indexed=False): + """Return standards-safe comparison coordinates with RA and Dec on separate lines.""" + if isinstance(vsp_params, dict): + vsp_params = [vsp_params] + + coordinates = [] + seen = set() + for vsp_param in vsp_params or []: + if not isinstance(vsp_param, dict) or vsp_param.get('ensemble_reference'): + continue + comp_ra = finite_float(vsp_param.get('comp_ra')) + comp_dec = finite_float(vsp_param.get('comp_dec')) + if not np.isfinite(comp_ra) or not np.isfinite(comp_dec): + continue + comparison_name = format_aavso_header_value(vsp_param.get('cname')) + identity = (comparison_name, round(float(comp_ra), 10), round(float(comp_dec), 10)) + if identity in seen: + continue + seen.add(identity) + coordinates.append((comparison_name, float(comp_ra), float(comp_dec))) + + if not coordinates: + return "" + if not indexed and len(coordinates) == 1: + _, comp_ra, comp_dec = coordinates[0] + return f"#COMPARISON_RA={comp_ra:.7f}\n#COMPARISON_DEC={comp_dec:.7f}\n" + + headers = [] + for index, (comparison_name, comp_ra, comp_dec) in enumerate(coordinates, start=1): + if comparison_name: + headers.append(f"#COMPARISON_{index}_NAME={comparison_name}") + headers.append(f"#COMPARISON_{index}_RA={comp_ra:.7f}") + headers.append(f"#COMPARISON_{index}_DEC={comp_dec:.7f}") + return "\n".join(headers) + "\n" + + +def aid_ensemble_comparison_metadata(vsp_param): + if not vsp_param or not vsp_param.get('ensemble_reference'): + return {} + + members = vsp_param.get('ensemble_members') + if isinstance(members, np.ndarray): + members = members.tolist() + if not isinstance(members, (list, tuple)) or not members: + labels = list(vsp_param.get('ensemble_member_labels') or []) + positions = list(vsp_param.get('ensemble_member_positions') or []) + ra_degs = list(vsp_param.get('ensemble_member_ra_degs') or []) + dec_degs = list(vsp_param.get('ensemble_member_dec_degs') or []) + magnitudes = list(vsp_param.get('ensemble_member_catalog_magnitudes') or []) + magnitude_errors = list(vsp_param.get('ensemble_member_catalog_errors') or []) + catalog_sources = list(vsp_param.get('ensemble_member_catalog_sources') or []) + member_count = max( + int(vsp_param.get('ensemble_member_count', 0) or 0), + len(labels), + len(ra_degs), + len(dec_degs), + ) + + def value_at(values, index): + return values[index] if index < len(values) else None + + members = [ + { + 'label': value_at(labels, index), + 'ra_deg': value_at(ra_degs, index), + 'dec_deg': value_at(dec_degs, index), + 'pixel_position': value_at(positions, index), + 'catalog_magnitude': value_at(magnitudes, index), + 'catalog_magnitude_error': value_at(magnitude_errors, index), + 'catalog_source': value_at(catalog_sources, index), + } + for index in range(member_count) + ] + else: + members = list(members) + + return prune_aavso_metadata({ + 'member_count': int(vsp_param.get('ensemble_member_count', len(members)) or len(members)), + 'members': members, + }) + + +def prune_aavso_metadata(value): + if isinstance(value, dict): + pruned = {} + for key, subvalue in value.items(): + cleaned = prune_aavso_metadata(subvalue) + if cleaned is None or cleaned == "" or cleaned == [] or cleaned == {}: + continue + pruned[key] = cleaned + return pruned + if isinstance(value, np.ndarray): + return prune_aavso_metadata(value.tolist()) + if isinstance(value, (list, tuple)): + return [ + cleaned for cleaned in (prune_aavso_metadata(item) for item in value) + if cleaned is not None and cleaned != "" and cleaned != [] and cleaned != {} + ] + return aavso_json_safe(value) + + +def format_aavso_json_header(name, payload, preserve_nulls=False): + payload = aavso_json_safe(payload) if preserve_nulls else prune_aavso_metadata(payload) + if not payload: + return "" + return f"#{name}={dumps(payload, sort_keys=True)}\n" + + +def aavso_result_entry(value, uncertainty=None, units=None): + value = finite_float(value) + uncertainty = finite_float(uncertainty) + if not np.isfinite(value): + return None + + if np.isfinite(uncertainty) and uncertainty >= 0: + value_text, uncertainty_text = format_value_and_uncertainty(value, uncertainty) + else: + value_text, uncertainty_text = str(round_to_2(value)), None + + entry = {'value': value_text} + if np.isfinite(uncertainty) and uncertainty >= 0: + entry['uncertainty'] = uncertainty_text + if units: + entry['units'] = units + return entry + + +def numeric_series_summary(values): + if values is None: + return {} + + try: + series = np.asarray(values, dtype=float).reshape(-1) + except (TypeError, ValueError): + return {} + + finite = series[np.isfinite(series)] + if finite.size == 0: + return {} + + return { + 'count': int(finite.size), + 'median': float(np.nanmedian(finite)), + 'std': float(np.nanstd(finite)), + 'min': float(np.nanmin(finite)), + 'max': float(np.nanmax(finite)), + } + + +def path_name(value): + if value is None: + return None + return Path(str(value)).name + + +def file_list_summary(files, limit=10): + files = list(files or []) + return { + 'count': len(files), + 'files': [path_name(file_name) for file_name in files[:limit]], + 'omitted_file_count': max(0, len(files) - limit), + } + + +def residual_scatter_fraction(fit): + transit_qc = getattr(fit, 'transit_qc', None) + if isinstance(transit_qc, dict): + qc_residual_scatter = finite_float(transit_qc.get('residual_scatter')) + if np.isfinite(qc_residual_scatter): + return qc_residual_scatter + + residuals = np.asarray(getattr(fit, 'residuals', np.array([])), dtype=float) + data = np.asarray(getattr(fit, 'data', np.array([])), dtype=float) + if residuals.size == 0 or data.size == 0: + return np.nan + + median_flux = np.nanmedian(data) + if not np.isfinite(median_flux) or median_flux == 0: + return np.nan + + if residuals.shape == data.shape: + return float(np.nanstd(residuals) / median_flux) + if residuals.size == 1: + return float(abs(residuals.reshape(-1)[0]) / median_flux) + return np.nan + + +def fit_data_model_uncertainty(fit): + data = np.asarray(getattr(fit, 'data', np.array([])), dtype=float) + if data.ndim != 1 or data.size == 0: + return None, None, None + + model = getattr(fit, 'model', None) + if model is None: + residuals = np.asarray(getattr(fit, 'residuals', np.array([])), dtype=float) + if residuals.shape == data.shape: + model = data - residuals + if model is None: + transit_model = getattr(fit, 'transit', None) + systematics_model = getattr(fit, 'airmass_model', None) + if transit_model is not None and systematics_model is not None: + model = np.asarray(transit_model, dtype=float) * np.asarray(systematics_model, dtype=float) + if model is None: + return data, None, None + + model = np.asarray(model, dtype=float) + if model.shape != data.shape: + return data, None, None + + uncertainty = getattr(fit, 'dataerr', None) + if uncertainty is not None: + uncertainty = np.asarray(uncertainty, dtype=float) + if uncertainty.shape != data.shape: + uncertainty = None + + return data, model, uncertainty + + +def infer_fit_quality_parameter_count(fit): + transit_qc = getattr(fit, 'transit_qc', None) + if isinstance(transit_qc, dict): + parameter_count = finite_float(transit_qc.get('transit_parameter_count')) + if np.isfinite(parameter_count) and parameter_count > 0: + return int(parameter_count) + + bounds = getattr(fit, 'bounds', None) + if isinstance(bounds, dict) and bounds: + return len(bounds) + + parameters = getattr(fit, 'parameters', None) + if isinstance(parameters, dict) and parameters: + return len(parameters) + + return 0 + + +def build_fit_quality_metadata(fit): + data, model, uncertainty = fit_data_model_uncertainty(fit) + if data is None or model is None: + return {} + + residuals = data - model + finite_mask = np.isfinite(data) & np.isfinite(model) & np.isfinite(residuals) + if not np.any(finite_mask): + return {} + + finite_residuals = residuals[finite_mask] + median_flux = np.nanmedian(data[finite_mask]) + rms_residual = float(np.sqrt(np.nanmean(finite_residuals ** 2))) + mad_residual = float(np.nanmedian(np.abs(finite_residuals))) + residual_scatter = ( + float(np.nanstd(finite_residuals) / median_flux) + if np.isfinite(median_flux) and median_flux != 0 + else np.nan + ) + point_count = int(np.count_nonzero(finite_mask)) + parameter_count = infer_fit_quality_parameter_count(fit) + degrees_of_freedom = point_count - parameter_count + + payload = { + 'point_count': point_count, + 'parameter_count': parameter_count, + 'degrees_of_freedom': degrees_of_freedom, + 'rms_residual': rms_residual, + 'rms_residual_percent': ( + 100.0 * rms_residual / median_flux + if np.isfinite(median_flux) and median_flux != 0 + else np.nan + ), + 'median_absolute_residual': mad_residual, + 'median_flux': float(median_flux) if np.isfinite(median_flux) else np.nan, + 'residual_scatter': residual_scatter, + 'residual_scatter_percent': 100.0 * residual_scatter if np.isfinite(residual_scatter) else np.nan, + 'uses_uncertainties': False, + } + + if uncertainty is None: + return payload + + uncertainty_mask = finite_mask & np.isfinite(uncertainty) & (uncertainty > 0) + if not np.any(uncertainty_mask): + return payload + + weighted_residuals = residuals[uncertainty_mask] + weighted_uncertainties = uncertainty[uncertainty_mask] + normalized_residuals = weighted_residuals / weighted_uncertainties + chi_square = float(np.sum(normalized_residuals ** 2)) + weighted_point_count = int(np.count_nonzero(uncertainty_mask)) + weighted_degrees_of_freedom = weighted_point_count - parameter_count + median_uncertainty = float(np.nanmedian(weighted_uncertainties)) + + payload.update({ + 'uses_uncertainties': True, + 'weighted_point_count': weighted_point_count, + 'degrees_of_freedom': weighted_degrees_of_freedom, + 'chi_square': chi_square, + 'reduced_chi_square': ( + chi_square / weighted_degrees_of_freedom + if weighted_degrees_of_freedom > 0 + else np.nan + ), + 'rms_normalized_residual': float(np.sqrt(np.nanmean(normalized_residuals ** 2))), + 'median_absolute_normalized_residual': float(np.nanmedian(np.abs(normalized_residuals))), + 'max_absolute_normalized_residual': float(np.nanmax(np.abs(normalized_residuals))), + 'median_uncertainty': median_uncertainty, + 'rms_residual_to_median_uncertainty': ( + rms_residual / median_uncertainty + if np.isfinite(median_uncertainty) and median_uncertainty > 0 + else np.nan + ), + }) + return payload + + +def red_noise_beta_factor(residual_fraction, coordinates=None, min_bin_size=2, max_bin_size=None): + residual_fraction = np.asarray(residual_fraction, dtype=float) + finite_mask = np.isfinite(residual_fraction) + + coordinates_array = None + if coordinates is not None: + coordinates_array = np.asarray(coordinates, dtype=float) + if coordinates_array.shape == residual_fraction.shape: + finite_mask &= np.isfinite(coordinates_array) + else: + coordinates_array = None + + residual_fraction = residual_fraction[finite_mask] + if coordinates_array is not None: + coordinates_array = coordinates_array[finite_mask] + sort_index = np.argsort(coordinates_array) + residual_fraction = residual_fraction[sort_index] + + point_count = int(residual_fraction.size) + payload = { + 'factor': 1.0, + 'point_count': point_count, + 'bin_sizes': [], + 'beta_by_bin': {}, + 'max_bin_size': np.nan, + } + min_bin_size = int(max(2, finite_float(min_bin_size, 2))) + if point_count < min_bin_size * 2: + return payload + + residual_fraction = residual_fraction - np.nanmedian(residual_fraction) + unbinned_rms = finite_float(np.nanstd(residual_fraction, ddof=1)) + if not np.isfinite(unbinned_rms) or unbinned_rms <= 0: + return payload + + if max_bin_size is None: + max_bin_size = min(10, max(min_bin_size, point_count // 4)) + max_bin_size = int(max(min_bin_size, finite_float(max_bin_size, min_bin_size))) + max_bin_size = min(max_bin_size, point_count // 2) + if max_bin_size < min_bin_size: + return payload + + beta_values = [] + for bin_size in range(min_bin_size, max_bin_size + 1): + bin_count = point_count // bin_size + if bin_count < 2: + continue + trimmed = residual_fraction[:bin_count * bin_size] + binned_means = np.nanmean(trimmed.reshape(bin_count, bin_size), axis=1) + binned_rms = finite_float(np.nanstd(binned_means, ddof=1)) + expected_rms = ( + unbinned_rms + / np.sqrt(bin_size) + * np.sqrt(bin_count / (bin_count - 1.0)) + ) + if not np.isfinite(binned_rms) or not np.isfinite(expected_rms) or expected_rms <= 0: + continue + beta = max(1.0, float(binned_rms / expected_rms)) + payload['bin_sizes'].append(bin_size) + payload['beta_by_bin'][str(bin_size)] = beta + beta_values.append(beta) + + if beta_values: + payload['factor'] = float(max(beta_values)) + payload['max_bin_size'] = max(payload['bin_sizes']) + return payload + + +def _fit_uncertainty_time_coordinates(fit, expected_shape): + for name in ('time', 'phase'): + values = getattr(fit, name, None) + if values is None: + continue + try: + values = np.asarray(values, dtype=float) + except (TypeError, ValueError): + continue + if values.shape == expected_shape: + return values + return None + + +def fit_empirical_transit_uncertainty(fit, fit_quality=None, transit_depth_threshold_fraction=0.05): + data, model, _ = fit_data_model_uncertainty(fit) + if data is None or model is None: + return {} + + data = np.asarray(data, dtype=float) + model = np.asarray(model, dtype=float) + if data.shape != model.shape or data.ndim != 1: + return {} + + residuals = data - model + finite_mask = np.isfinite(data) & np.isfinite(model) & np.isfinite(residuals) + if not np.any(finite_mask): + return {} + + fit_quality = fit_quality or {} + median_flux = np.nanmedian(data[finite_mask]) + residual_scatter = finite_float(fit_quality.get('residual_scatter')) + if not np.isfinite(residual_scatter): + if np.isfinite(median_flux) and median_flux != 0: + residual_scatter = float(np.nanstd(residuals[finite_mask]) / median_flux) + if not np.isfinite(residual_scatter) or residual_scatter < 0: + return {} + + transit = np.asarray(getattr(fit, 'transit', np.array([])), dtype=float) + if transit.shape != data.shape: + return {} + + transit_mask = finite_mask & np.isfinite(transit) + if np.count_nonzero(transit_mask) < 2: + return {} + + transit_values = transit[transit_mask] + baseline = finite_float(np.nanpercentile(transit_values, 95)) + if not np.isfinite(baseline): + return {} + + transit_depth_profile = baseline - transit + max_depth = finite_float(np.nanmax(transit_depth_profile[transit_mask])) + if not np.isfinite(max_depth) or max_depth <= 0: + return {} + + threshold_fraction = finite_float(transit_depth_threshold_fraction, 0.05) + if not np.isfinite(threshold_fraction) or threshold_fraction <= 0: + threshold_fraction = 0.05 + depth_threshold = max_depth * threshold_fraction + in_transit_mask = transit_mask & (transit_depth_profile >= depth_threshold) + out_of_transit_mask = transit_mask & (transit_depth_profile < depth_threshold) + + in_transit_count = int(np.count_nonzero(in_transit_mask)) + out_of_transit_count = int(np.count_nonzero(out_of_transit_mask)) + if in_transit_count <= 0: + return {} + + sample_term = 1.0 / in_transit_count + if out_of_transit_count > 0: + sample_term += 1.0 / out_of_transit_count + + depth_standard_error_fraction = float(residual_scatter * np.sqrt(sample_term)) + if not np.isfinite(depth_standard_error_fraction) or depth_standard_error_fraction < 0: + return {} + if out_of_transit_count > 0: + baseline_standard_error_fraction = float(residual_scatter / np.sqrt(out_of_transit_count)) + else: + baseline_standard_error_fraction = float(depth_standard_error_fraction) + + residual_fraction = residuals + if np.isfinite(median_flux) and median_flux != 0: + residual_fraction = residuals / median_flux + coordinates = _fit_uncertainty_time_coordinates(fit, data.shape) + max_beta_bin_size = min(10, max(2, in_transit_count // 4)) + beta_payload = red_noise_beta_factor( + residual_fraction[finite_mask], + coordinates=coordinates[finite_mask] if coordinates is not None else None, + min_bin_size=2, + max_bin_size=max_beta_bin_size, + ) + red_noise_beta = finite_float(beta_payload.get('factor'), 1.0) + if not np.isfinite(red_noise_beta) or red_noise_beta < 1.0: + red_noise_beta = 1.0 + depth_uncertainty_fraction = float(depth_standard_error_fraction * red_noise_beta) + baseline_uncertainty_fraction = float(baseline_standard_error_fraction * red_noise_beta) + depth_flux_scatter_fraction = float(residual_scatter) + + parameters = getattr(fit, 'parameters', {}) or {} + errors = getattr(fit, 'errors', {}) or {} + rprs = finite_float(parameters.get('rprs')) + model_rprs_uncertainty = finite_float(errors.get('rprs')) + rprs_prior_fallback = bool(getattr(fit, 'rprs_prior_fallback_applied', False)) + if rprs_prior_fallback: + model_rprs_uncertainty = np.nan + data_rprs_uncertainty = np.nan + data_rprs_standard_error = np.nan + data_rprs_flux_scatter_uncertainty = np.nan + combined_rprs_uncertainty = np.nan + combined_rprs_standard_error = np.nan + conservative_rprs_uncertainty = np.nan + if np.isfinite(rprs) and rprs > 0: + data_rprs_uncertainty = float(depth_uncertainty_fraction / (2.0 * rprs)) + data_rprs_standard_error = float(depth_standard_error_fraction / (2.0 * rprs)) + data_rprs_flux_scatter_uncertainty = float(depth_flux_scatter_fraction / (2.0 * rprs)) + if rprs_prior_fallback: + combined_rprs_uncertainty = data_rprs_uncertainty + combined_rprs_standard_error = data_rprs_standard_error + conservative_rprs_uncertainty = data_rprs_uncertainty + elif np.isfinite(model_rprs_uncertainty) and model_rprs_uncertainty >= 0: + combined_rprs_uncertainty = float( + np.sqrt(model_rprs_uncertainty ** 2 + data_rprs_uncertainty ** 2) + ) + combined_rprs_standard_error = float( + np.sqrt(model_rprs_uncertainty ** 2 + data_rprs_standard_error ** 2) + ) + conservative_rprs_uncertainty = float( + max(model_rprs_uncertainty, data_rprs_uncertainty) + ) + + return { + 'available': True, + 'residual_scatter': residual_scatter, + 'residual_scatter_percent': residual_scatter * 100.0, + 'in_transit_point_count': in_transit_count, + 'out_of_transit_point_count': out_of_transit_count, + 'transit_depth_threshold_fraction': threshold_fraction, + 'model_depth_fraction': max_depth, + 'model_depth_percent': max_depth * 100.0, + 'depth_uncertainty_fraction': depth_uncertainty_fraction, + 'depth_uncertainty_percent': depth_uncertainty_fraction * 100.0, + 'depth_red_noise_uncertainty_fraction': depth_uncertainty_fraction, + 'depth_red_noise_uncertainty_percent': depth_uncertainty_fraction * 100.0, + 'baseline_standard_error_fraction': baseline_standard_error_fraction, + 'baseline_standard_error_percent': baseline_standard_error_fraction * 100.0, + 'baseline_red_noise_uncertainty_fraction': baseline_uncertainty_fraction, + 'baseline_red_noise_uncertainty_percent': baseline_uncertainty_fraction * 100.0, + 'depth_flux_scatter_fraction': depth_flux_scatter_fraction, + 'depth_flux_scatter_percent': depth_flux_scatter_fraction * 100.0, + 'depth_standard_error_fraction': depth_standard_error_fraction, + 'depth_standard_error_percent': depth_standard_error_fraction * 100.0, + 'red_noise_beta_factor': red_noise_beta, + 'red_noise_beta_bin_sizes': beta_payload.get('bin_sizes', []), + 'red_noise_beta_by_bin': beta_payload.get('beta_by_bin', {}), + 'red_noise_beta_max_bin_size': beta_payload.get('max_bin_size', np.nan), + 'rprs': rprs, + 'model_rprs_uncertainty': model_rprs_uncertainty, + 'data_rprs_uncertainty': data_rprs_uncertainty, + 'data_rprs_red_noise_uncertainty': data_rprs_uncertainty, + 'data_rprs_standard_error': data_rprs_standard_error, + 'data_rprs_flux_scatter_uncertainty': data_rprs_flux_scatter_uncertainty, + 'combined_rprs_uncertainty': combined_rprs_uncertainty, + 'combined_rprs_red_noise_uncertainty': combined_rprs_uncertainty, + 'combined_rprs_standard_error': combined_rprs_standard_error, + 'conservative_rprs_uncertainty': conservative_rprs_uncertainty, + 'rprs_uncertainty_basis': ( + 'prior_assumed_data_only' + if rprs_prior_fallback + else 'model_plus_red_noise' + ), + 'rprs_prior_fallback_applied': rprs_prior_fallback, + 'rprs_prior_fallback_prior_value': finite_float( + getattr(fit, 'rprs_prior_fallback_prior_value', np.nan) + ), + 'rprs_prior_fallback_original_fit_value': finite_float( + getattr(fit, 'rprs_prior_fallback_original_fit_value', np.nan) + ), + 'rprs_prior_fallback_data_uncertainty': finite_float( + getattr(fit, 'rprs_prior_fallback_data_uncertainty', np.nan) + ), + 'rprs_prior_fallback_note': getattr(fit, 'rprs_prior_fallback_note', None), + } + + +def photometry_method_from_info(photometry_info): + if not isinstance(photometry_info, dict): + return None + + min_aperture = photometry_info.get('min_aperture') + min_aperture = finite_float(min_aperture) + if not np.isfinite(min_aperture): + return None + if min_aperture == 0: + return "PSF photometry" + if min_aperture < 0: + return "Aperture photometry without comparison star" + return "Aperture photometry" + + +def build_aavso_qc_metadata(fit): + transit_qc = getattr(fit, 'transit_qc', None) + if not isinstance(transit_qc, dict): + return {} + + fields = ( + 'computed', 'status', 'summary', 'preferred_model', 'point_count', + 'transit_chi2', 'flat_chi2', 'delta_chi2', 'transit_bic', 'flat_bic', + 'delta_bic', 'transit_parameter_count', 'flat_parameter_count', + 'flat_baseline', 'flat_a2', 'flat_model_note', 'residual_scatter', + 'transit_depth_for_residual_scatter', 'residual_scatter_to_depth_ratio', + 'residual_flatness_score', 'residual_flatness_trend_strength', + 'residual_flatness_curve_strength', 'residual_flatness_scatter_ratio', + 'residual_flatness_zero_offset_strength', + 'residual_flatness_sign_imbalance', 'residual_flatness_zero_bias_score', + 'residual_flatness_trend_score', 'residual_flatness_curve_score', + 'residual_flatness_scatter_stability_score', + 'residual_flatness_dominant_metric', 'residual_flatness_detail', + 'tmid_gaussianity_score', 'tmid_gaussianity_score_uncertainty', + 'tmid_gaussianity_gaussian_distance', 'tmid_gaussianity_flat_distance', + 'tmid_gaussianity_effective_sample_count', 'tmid_gaussianity_sample_count', + 'tmid_gaussianity_robust_sigma', 'tmid_gaussianity_detail', + 'rprs_sigma', 'duration_ratio', 'eebls_depth_snr', + 'sampling_score', 'sampling_detail', 'sampling_ingress_count', + 'sampling_egress_count', 'sampling_in_transit_count', + 'sampling_pre_baseline_count', 'sampling_post_baseline_count', + 'sampling_total_duration', 'sampling_ingress_duration', + 'use_deviation_from_expected_transit_in_qc', 'deviation_sigma_threshold', + 'expected_tmid', 'expected_tmid_unc', 'fitted_tmid', + 'expected_rprs', 'expected_rprs_unc', 'fitted_rprs', 'fitted_rprs_unc', + 'rprs_deviation_fit_unc', 'rprs_deviation_model_fit_unc', + 'rprs_deviation_data_fit_unc', 'rprs_deviation_combined_fit_unc', + 'rprs_deviation_expected_unc', + 'rprs_deviation_systematic_floor', 'rprs_deviation_unc', + 'rprs_deviation_sigma', 'rprs_deviation_score', + 'rprs_prior_assumed', 'rprs_prior_assumed_note', + 'deviation_from_expected_value', 'ktmf_metric', 'ktmf_contributions', + 'notes', + ) + return {field: transit_qc.get(field) for field in fields if field in transit_qc} + + +def compact_ktmf_contributions(contributions): + compact = [] + for contribution in contributions or []: + if not isinstance(contribution, dict): + continue + compact.append({ + 'label': contribution.get('label'), + 'available': contribution.get('available'), + 'points': contribution.get('points'), + 'max_points': contribution.get('max_points'), + 'score': contribution.get('score'), + 'score_uncertainty': contribution.get('score_uncertainty'), + 'detail': contribution.get('detail'), + }) + return compact + + +def compact_comparison_attempt_decision(attempt): + if not isinstance(attempt, dict): + return {} + + comp_index = attempt.get('comp_index') + try: + comp_number = int(comp_index) + 1 + except (TypeError, ValueError): + comp_number = None + + return { + 'rank': attempt.get('rank'), + 'comparison_star': comp_number, + 'label': attempt.get('label'), + 'selected': attempt.get('selected'), + 'selection_reason': attempt.get('selection_reason'), + 'selection_pass_ktmf_metric': attempt.get('selection_pass_ktmf_metric'), + 'selection_pass_transit_delta_bic': attempt.get('selection_pass_transit_delta_bic'), + 'selection_pass_eebls_snr': attempt.get('selection_pass_eebls_snr'), + 'selection_pass_residual_scatter': attempt.get('selection_pass_residual_scatter'), + 'target_model_scatter_basis': attempt.get('target_model_scatter_basis'), + 'projected_full_residual_scatter': attempt.get('projected_full_residual_scatter'), + 'selection_scatter': attempt.get('selection_scatter'), + 'selection_scatter_basis': attempt.get('selection_scatter_basis'), + 'target_comp_scatter': attempt.get('target_comp_scatter'), + 'selection_pass_target_comp_scatter': attempt.get('selection_pass_target_comp_scatter'), + 'selection_pass_transit_qc_status': attempt.get('selection_pass_transit_qc_status'), + 'selection_pass_transit_qc_summary': attempt.get('selection_pass_transit_qc_summary'), + 'scatter_gate_passed': attempt.get('scatter_gate_passed'), + 'scatter_gate_lowest_residual_scatter': attempt.get('scatter_gate_lowest_residual_scatter'), + 'scatter_gate_threshold': attempt.get('scatter_gate_threshold'), + 'scatter_adjusted_ktmf_metric': attempt.get('scatter_adjusted_ktmf_metric'), + 'combined_quality_ktmf_metric': attempt.get('combined_quality_ktmf_metric'), + 'combined_quality_best_residual_scatter': attempt.get('combined_quality_best_residual_scatter'), + 'combined_quality_best_target_comp_scatter': attempt.get('combined_quality_best_target_comp_scatter'), + 'combined_quality_best_comp_stability': attempt.get('combined_quality_best_comp_stability'), + 'final_refit_metric_note': attempt.get('final_refit_metric_note'), + 'ktmf_metric': attempt.get('ktmf_metric'), + 'ktmf_contributions': compact_ktmf_contributions(attempt.get('ktmf_contributions')), + 'transit_delta_bic': attempt.get('transit_delta_bic'), + 'eebls_snr': attempt.get('eebls_snr'), + 'residual_scatter': attempt.get('residual_scatter'), + 'fit_point_count': attempt.get('fit_point_count'), + 'transit_qc_status': attempt.get('transit_qc_status'), + 'transit_qc_summary': attempt.get('transit_qc_summary'), + 'rejected_by_transit_qc': attempt.get('rejected_by_transit_qc'), + 'failure_reason': attempt.get('failure_reason'), + } + + +def compact_comparison_attempt_decisions(attempts, limit=10): + attempts = list(attempts or []) + return { + 'candidate_count': len(attempts), + 'candidates': [ + compact_comparison_attempt_decision(attempt) + for attempt in attempts[:limit] + ], + 'omitted_candidate_count': max(0, len(attempts) - limit), + } + + +def build_ktmf_decision_metadata(fit, photometry_info=None): + transit_qc = getattr(fit, 'transit_qc', None) + payload = {} + if isinstance(transit_qc, dict): + payload['target_fit'] = { + 'status': transit_qc.get('status'), + 'summary': transit_qc.get('summary'), + 'ktmf_metric': transit_qc.get('ktmf_metric'), + 'ktmf_contributions': compact_ktmf_contributions(transit_qc.get('ktmf_contributions')), + 'delta_bic': transit_qc.get('delta_bic'), + 'delta_chi2': transit_qc.get('delta_chi2'), + 'eebls_depth_snr': transit_qc.get('eebls_depth_snr'), + 'residual_scatter': transit_qc.get('residual_scatter'), + 'transit_depth_for_residual_scatter': transit_qc.get('transit_depth_for_residual_scatter'), + 'residual_scatter_to_depth_ratio': transit_qc.get('residual_scatter_to_depth_ratio'), + 'residual_flatness_score': transit_qc.get('residual_flatness_score'), + 'residual_flatness_detail': transit_qc.get('residual_flatness_detail'), + 'residual_flatness_trend_strength': transit_qc.get('residual_flatness_trend_strength'), + 'residual_flatness_curve_strength': transit_qc.get('residual_flatness_curve_strength'), + 'residual_flatness_scatter_ratio': transit_qc.get('residual_flatness_scatter_ratio'), + 'residual_flatness_zero_offset_strength': transit_qc.get('residual_flatness_zero_offset_strength'), + 'residual_flatness_sign_imbalance': transit_qc.get('residual_flatness_sign_imbalance'), + 'residual_flatness_zero_bias_score': transit_qc.get('residual_flatness_zero_bias_score'), + 'sampling_score': transit_qc.get('sampling_score'), + 'sampling_detail': transit_qc.get('sampling_detail'), + 'sampling_ingress_count': transit_qc.get('sampling_ingress_count'), + 'sampling_egress_count': transit_qc.get('sampling_egress_count'), + 'sampling_in_transit_count': transit_qc.get('sampling_in_transit_count'), + 'sampling_pre_baseline_count': transit_qc.get('sampling_pre_baseline_count'), + 'sampling_post_baseline_count': transit_qc.get('sampling_post_baseline_count'), + 'deviation_from_expected_value': transit_qc.get('deviation_from_expected_value'), + 'rprs_deviation_fit_unc': transit_qc.get('rprs_deviation_fit_unc'), + 'rprs_deviation_model_fit_unc': transit_qc.get('rprs_deviation_model_fit_unc'), + 'rprs_deviation_data_fit_unc': transit_qc.get('rprs_deviation_data_fit_unc'), + 'rprs_deviation_expected_unc': transit_qc.get('rprs_deviation_expected_unc'), + 'rprs_deviation_unc': transit_qc.get('rprs_deviation_unc'), + } + + if isinstance(photometry_info, dict): + selected_attempt = photometry_info.get('selected_comparison_attempt') + selected_payload = compact_comparison_attempt_decision(selected_attempt) + if not selected_payload: + selected_payload = { + 'comparison_star': photometry_info.get('comp_star_num'), + 'selected': photometry_info.get('comp_star_num') is not None, + 'selection_reason': photometry_info.get('selected_comparison_selection_reason'), + 'ktmf_metric': photometry_info.get('comparison_ktmf_metric'), + 'ktmf_contributions': compact_ktmf_contributions( + photometry_info.get('selected_comparison_ktmf_contributions') + ), + 'transit_delta_bic': photometry_info.get('comparison_transit_delta_bic'), + 'eebls_snr': photometry_info.get('comparison_eebls_snr'), + 'fit_point_count': photometry_info.get('selected_comparison_fit_point_count'), + 'transit_qc_status': photometry_info.get('selected_comparison_transit_qc_status'), + 'transit_qc_summary': photometry_info.get('selected_comparison_transit_qc_summary'), + } + + payload['comparison_selection'] = { + 'basis': photometry_info.get('selection_basis'), + 'metric': photometry_info.get('selection_metric'), + 'field_score': photometry_info.get('calibration_field_score'), + 'selected': selected_payload, + } + + attempt_summary = compact_comparison_attempt_decisions( + photometry_info.get('comparison_fit_attempt_summaries') + ) + if attempt_summary['candidate_count']: + payload['comparison_selection'].update(attempt_summary) + + return payload + + +def format_ktmf_metric(value): + value = finite_float(value) + return f"{value:.2f} / 5.00" if np.isfinite(value) else "n/a" + + +def format_ktmf_status(value): + value = finite_float(value) + if not np.isfinite(value): + return "UNKNOWN" + if value >= 4.0: + return "PASS" + if value >= 3.0: + return "MARGINAL" + return "FAIL" + + +def format_transit_qc_headline_final_params(transit_qc): + params = {} + if not isinstance(transit_qc, dict) or not transit_qc: + return params + + qc_status = transit_qc.get('status') + if qc_status: + params["Transit detection QC"] = str(qc_status).upper() + + qc_ktmf = finite_float(transit_qc.get('ktmf_metric')) + if np.isfinite(qc_ktmf): + params["KTMF"] = f"{qc_ktmf:.2f} / 5.00" + + return params + + +def format_optional_metric(label, value, precision=2): + value = finite_float(value) + if not np.isfinite(value): + return None + return f"{label}={value:.{precision}f}" + + +def format_transit_delta_bic(value): + value = finite_float(value) + return f"{value:.2f}" if np.isfinite(value) else "n/a" + + +def format_percent_metric(label, value, precision=4): + value = finite_float(value) + if not np.isfinite(value): + return None + return f"{label}={value * 100.0:.{precision}f}%" + + +def metric_values_differ(first_value, second_value, tolerance=5.0e-3): + first_value = finite_float(first_value) + second_value = finite_float(second_value) + return ( + np.isfinite(first_value) + and np.isfinite(second_value) + and abs(first_value - second_value) > tolerance + ) + + +def format_ktmf_candidate_decision(attempt): + attempt = compact_comparison_attempt_decision(attempt) + label = attempt.get('label') or ( + f"Comp {attempt['comparison_star']}" if attempt.get('comparison_star') is not None else "Comparison candidate" + ) + selected_text = " [selected]" if attempt.get('selected') else "" + ktmf_text = f"KTMF={format_ktmf_metric(attempt.get('ktmf_metric'))}" + if metric_values_differ( + attempt.get('selection_pass_ktmf_metric'), + attempt.get('ktmf_metric'), + ): + ktmf_text += ( + f" (selection-pass {format_ktmf_metric(attempt.get('selection_pass_ktmf_metric'))})" + ) + parts = [f"{label}{selected_text}: {ktmf_text}"] + for metric_text in ( + format_optional_metric("Delta BIC", attempt.get('transit_delta_bic')), + format_optional_metric("EEBLS SNR", attempt.get('eebls_snr')), + format_percent_metric("Target/comp scatter", attempt.get('target_comp_scatter')), + format_percent_metric("Target model scatter", attempt.get('residual_scatter')), + format_percent_metric("Selection scatter", attempt.get('selection_scatter')), + format_optional_metric("KTMF/projected-scatter score", attempt.get('combined_quality_ktmf_metric')), + ): + if metric_text: + parts.append(metric_text) + target_model_basis = attempt.get('target_model_scatter_basis') + selection_scatter_basis = attempt.get('selection_scatter_basis') + if target_model_basis: + parts.append(f"target model scatter basis={target_model_basis}") + if selection_scatter_basis: + parts.append(f"selection scatter basis={selection_scatter_basis}") + selection_pass_target_comp_scatter = finite_float(attempt.get('selection_pass_target_comp_scatter')) + target_comp_scatter = finite_float(attempt.get('target_comp_scatter')) + if ( + np.isfinite(selection_pass_target_comp_scatter) + and ( + not np.isfinite(target_comp_scatter) + or abs(selection_pass_target_comp_scatter - target_comp_scatter) > 1.0e-5 + ) + ): + parts.append( + "selection-pass target/comp scatter=" + f"{selection_pass_target_comp_scatter * 100.0:.4f}%" + ) + if metric_values_differ( + attempt.get('selection_pass_residual_scatter'), + attempt.get('residual_scatter'), + tolerance=1.0e-5, + ): + parts.append( + "selection-pass target model scatter=" + f"{finite_float(attempt.get('selection_pass_residual_scatter')) * 100.0:.4f}%" + ) + if metric_values_differ( + attempt.get('selection_pass_transit_delta_bic'), + attempt.get('transit_delta_bic'), + tolerance=1.0e-2, + ): + parts.append( + "selection-pass Delta BIC=" + f"{format_transit_delta_bic(attempt.get('selection_pass_transit_delta_bic'))}" + ) + if metric_values_differ( + attempt.get('selection_pass_eebls_snr'), + attempt.get('eebls_snr'), + tolerance=1.0e-2, + ): + parts.append( + "selection-pass EEBLS SNR=" + f"{finite_float(attempt.get('selection_pass_eebls_snr')):.2f}" + ) + qc_status = attempt.get('transit_qc_status') + if qc_status: + parts.append(f"QC={str(qc_status).upper()}") + reason = attempt.get('selection_reason') or attempt.get('failure_reason') + if reason: + parts.append(f"reason={reason}") + return ", ".join(parts) + + +def format_ktmf_decision_final_params(fit, photometry_info=None): + params = {} + + transit_qc = getattr(fit, 'transit_qc', None) + if isinstance(transit_qc, dict): + ktmf_metric = finite_float(transit_qc.get('ktmf_metric')) + if np.isfinite(ktmf_metric): + target_status = format_ktmf_status(ktmf_metric) + params["KTMF target-fit decision"] = ( + f"{target_status}: KTMF={format_ktmf_metric(ktmf_metric)}" + ) + for contribution_index, contribution in enumerate( + compact_ktmf_contributions(transit_qc.get('ktmf_contributions')), + start=1, + ): + label = contribution.get('label', f'Component {contribution_index}') + available = bool(contribution.get('available')) + points = finite_float(contribution.get('points'), 0.0) + max_points = finite_float(contribution.get('max_points'), 0.0) + score = finite_float(contribution.get('score')) + detail = contribution.get('detail') or 'n/a' + if available and np.isfinite(score): + params[f"KTMF target contribution {contribution_index}"] = ( + f"{label}: +{points:.2f}/{max_points:.2f} (score={score:.2f}; {detail})" + ) + else: + params[f"KTMF target contribution {contribution_index}"] = ( + f"{label}: +0.00/0.00 (unavailable; {detail})" + ) + + if not isinstance(photometry_info, dict): + return params + + basis = photometry_info.get('selection_basis') + metric = photometry_info.get('selection_metric') + if basis or metric: + params["KTMF comparison selection mode"] = ( + f"basis={basis or 'n/a'}, metric={metric or 'n/a'}" + ) + + selected_attempt = photometry_info.get('selected_comparison_attempt') + if selected_attempt: + params["KTMF selected comparison decision"] = format_ktmf_candidate_decision(selected_attempt) + elif photometry_info.get('comp_star_num') is not None: + selected_payload = { + 'label': f"Comp {photometry_info.get('comp_star_num')}", + 'selected': True, + 'selection_reason': photometry_info.get('selected_comparison_selection_reason'), + 'ktmf_metric': photometry_info.get('comparison_ktmf_metric'), + 'ktmf_contributions': photometry_info.get('selected_comparison_ktmf_contributions'), + 'transit_delta_bic': photometry_info.get('comparison_transit_delta_bic'), + 'eebls_snr': photometry_info.get('comparison_eebls_snr'), + 'transit_qc_status': photometry_info.get('selected_comparison_transit_qc_status'), + } + params["KTMF selected comparison decision"] = format_ktmf_candidate_decision(selected_payload) + + for contribution_index, contribution in enumerate( + compact_ktmf_contributions(photometry_info.get('selected_comparison_ktmf_contributions')), + start=1, + ): + label = contribution.get('label', f'Component {contribution_index}') + available = bool(contribution.get('available')) + points = finite_float(contribution.get('points'), 0.0) + max_points = finite_float(contribution.get('max_points'), 0.0) + score = finite_float(contribution.get('score')) + detail = contribution.get('detail') or 'n/a' + if available and np.isfinite(score): + params[f"KTMF selected comparison contribution {contribution_index}"] = ( + f"{label}: +{points:.2f}/{max_points:.2f} (score={score:.2f}; {detail})" + ) + else: + params[f"KTMF selected comparison contribution {contribution_index}"] = ( + f"{label}: +0.00/0.00 (unavailable; {detail})" + ) + + for attempt_index, attempt in enumerate( + (photometry_info.get('comparison_fit_attempt_summaries') or [])[:10], + start=1, + ): + params[f"KTMF comparison candidate {attempt_index}"] = format_ktmf_candidate_decision(attempt) + + return params + + +def format_fit_quality_final_params(fit_quality): + fit_quality = fit_quality or {} + params = {} + + reduced_chi_square = finite_float(fit_quality.get('reduced_chi_square')) + if np.isfinite(reduced_chi_square): + params["Fit quality reduced chi-square"] = f"{reduced_chi_square:.3f}" + + chi_square = finite_float(fit_quality.get('chi_square')) + if np.isfinite(chi_square): + params["Fit quality chi-square"] = f"{chi_square:.2f}" + + degrees_of_freedom = fit_quality.get('degrees_of_freedom') + try: + degrees_of_freedom = int(degrees_of_freedom) + except (TypeError, ValueError): + degrees_of_freedom = None + if degrees_of_freedom is not None: + params["Fit quality degrees of freedom"] = str(degrees_of_freedom) + + rms_residual_percent = finite_float(fit_quality.get('rms_residual_percent')) + if np.isfinite(rms_residual_percent): + params["Fit quality RMS residual"] = f"{rms_residual_percent:.4f} %" + + median_abs_normalized_residual = finite_float( + fit_quality.get('median_absolute_normalized_residual') + ) + if np.isfinite(median_abs_normalized_residual): + params["Fit quality median absolute normalized residual"] = ( + f"{median_abs_normalized_residual:.2f} sigma" + ) + + rms_uncertainty_ratio = finite_float(fit_quality.get('rms_residual_to_median_uncertainty')) + if np.isfinite(rms_uncertainty_ratio): + params["Fit quality RMS residual / median uncertainty"] = f"{rms_uncertainty_ratio:.2f}" + + point_count = fit_quality.get('weighted_point_count', fit_quality.get('point_count')) + try: + point_count = int(point_count) + except (TypeError, ValueError): + point_count = None + if point_count is not None: + params["Fit quality point count"] = str(point_count) + + if fit_quality and not fit_quality.get('uses_uncertainties'): + params["Fit quality note"] = "Per-point uncertainties unavailable; chi-square metrics not reported." + + return params + + +def format_empirical_transit_uncertainty_final_params(empirical_uncertainty): + empirical_uncertainty = empirical_uncertainty or {} + if not empirical_uncertainty.get('available'): + return {} + + params = {} + rprs = finite_float(empirical_uncertainty.get('rprs')) + model_rprs_uncertainty = finite_float(empirical_uncertainty.get('model_rprs_uncertainty')) + data_rprs_uncertainty = finite_float(empirical_uncertainty.get('data_rprs_uncertainty')) + combined_rprs_uncertainty = finite_float(empirical_uncertainty.get('combined_rprs_uncertainty')) + conservative_rprs_uncertainty = finite_float( + empirical_uncertainty.get('conservative_rprs_uncertainty') + ) + data_rprs_standard_error = finite_float(empirical_uncertainty.get('data_rprs_standard_error')) + data_rprs_flux_scatter_uncertainty = finite_float( + empirical_uncertainty.get('data_rprs_flux_scatter_uncertainty') + ) + combined_rprs_standard_error = finite_float( + empirical_uncertainty.get('combined_rprs_standard_error') + ) + depth_uncertainty_percent = finite_float(empirical_uncertainty.get('depth_uncertainty_percent')) + depth_flux_scatter_percent = finite_float( + empirical_uncertainty.get('depth_flux_scatter_percent') + ) + depth_standard_error_percent = finite_float( + empirical_uncertainty.get('depth_standard_error_percent') + ) + baseline_red_noise_percent = finite_float( + empirical_uncertainty.get('baseline_red_noise_uncertainty_percent') + ) + baseline_standard_error_percent = finite_float( + empirical_uncertainty.get('baseline_standard_error_percent') + ) + residual_scatter_percent = finite_float(empirical_uncertainty.get('residual_scatter_percent')) + red_noise_beta = finite_float(empirical_uncertainty.get('red_noise_beta_factor')) + rprs_prior_fallback = bool(empirical_uncertainty.get('rprs_prior_fallback_applied')) + rprs_uncertainty_basis = empirical_uncertainty.get('rprs_uncertainty_basis') + + if ( + not rprs_prior_fallback + and np.isfinite(rprs) + and np.isfinite(model_rprs_uncertainty) + and model_rprs_uncertainty >= 0 + ): + params["Ratio of Planet to Stellar Radius (Rp/R*) model-fit uncertainty"] = ( + format_value_with_uncertainty(rprs, model_rprs_uncertainty) + ) + if np.isfinite(rprs) and np.isfinite(data_rprs_uncertainty) and data_rprs_uncertainty >= 0: + if rprs_prior_fallback: + params["Ratio of Planet to Stellar Radius (Rp/R*) prior-assumed data-only uncertainty"] = ( + format_value_with_uncertainty(rprs, data_rprs_uncertainty) + ) + else: + params["Ratio of Planet to Stellar Radius (Rp/R*) data-fit red-noise uncertainty"] = ( + format_value_with_uncertainty(rprs, data_rprs_uncertainty) + ) + if np.isfinite(rprs) and np.isfinite(combined_rprs_uncertainty) and combined_rprs_uncertainty >= 0: + if rprs_prior_fallback: + params["Ratio of Planet to Stellar Radius (Rp/R*) data-only uncertainty used for primary value"] = ( + format_value_with_uncertainty(rprs, combined_rprs_uncertainty) + ) + else: + params["Ratio of Planet to Stellar Radius (Rp/R*) model+red-noise uncertainty"] = ( + format_value_with_uncertainty(rprs, combined_rprs_uncertainty) + ) + if np.isfinite(conservative_rprs_uncertainty): + params["Conservative Rp/R* uncertainty to quote"] = ( + f"+/- {format_uncertainty(conservative_rprs_uncertainty)}" + ) + if np.isfinite(rprs) and np.isfinite(data_rprs_standard_error) and data_rprs_standard_error >= 0: + params["Ratio of Planet to Stellar Radius (Rp/R*) data-fit standard-error estimate"] = ( + format_value_with_uncertainty(rprs, data_rprs_standard_error) + ) + if ( + not rprs_prior_fallback + and np.isfinite(rprs) + and np.isfinite(combined_rprs_standard_error) + and combined_rprs_standard_error >= 0 + ): + params["Ratio of Planet to Stellar Radius (Rp/R*) model+standard-error estimate"] = ( + format_value_with_uncertainty(rprs, combined_rprs_standard_error) + ) + if ( + np.isfinite(rprs) + and np.isfinite(data_rprs_flux_scatter_uncertainty) + and data_rprs_flux_scatter_uncertainty >= 0 + ): + params["Ratio of Planet to Stellar Radius (Rp/R*) flux-scatter equivalent"] = ( + format_value_with_uncertainty(rprs, data_rprs_flux_scatter_uncertainty) + ) + if np.isfinite(depth_uncertainty_percent): + params["Transit depth red-noise uncertainty"] = ( + f"+/- {depth_uncertainty_percent:.4f} %" + ) + if np.isfinite(depth_flux_scatter_percent): + params["Transit depth flux-scatter equivalent"] = ( + f"+/- {depth_flux_scatter_percent:.4f} %" + ) + if np.isfinite(depth_standard_error_percent): + params["Transit depth data-fit standard-error estimate"] = ( + f"+/- {depth_standard_error_percent:.4f} %" + ) + if np.isfinite(baseline_red_noise_percent): + params["Flux baseline red-noise uncertainty"] = ( + f"+/- {baseline_red_noise_percent:.4f} %" + ) + if np.isfinite(baseline_standard_error_percent): + params["Flux baseline standard-error estimate"] = ( + f"+/- {baseline_standard_error_percent:.4f} %" + ) + if np.isfinite(red_noise_beta): + params["Red-noise beta factor"] = f"{red_noise_beta:.3f}" + if rprs_uncertainty_basis: + params["Rp/R* uncertainty basis"] = str(rprs_uncertainty_basis) + fallback_note = empirical_uncertainty.get('rprs_prior_fallback_note') + if fallback_note: + params["Rp/R* prior fallback note"] = str(fallback_note) + + beta_bins = empirical_uncertainty.get('red_noise_beta_bin_sizes') + if beta_bins: + params["Red-noise beta bin sizes"] = ", ".join(str(int(item)) for item in beta_bins) + + in_count = empirical_uncertainty.get('in_transit_point_count') + out_count = empirical_uncertainty.get('out_of_transit_point_count') + try: + in_count = int(in_count) + out_count = int(out_count) + except (TypeError, ValueError): + in_count = None + out_count = None + if in_count is not None and out_count is not None: + params["Data-fit uncertainty point counts"] = ( + f"{in_count} in transit, {out_count} out of transit" + ) + + if np.isfinite(residual_scatter_percent): + params["Flux residual scatter around model"] = ( + f"{residual_scatter_percent:.4f} %" + ) + if rprs_prior_fallback: + params["Uncertainty interpretation note"] = ( + "The primary Rp/R* and radius-ratio area-depth uncertainties use the input " + "prior Rp/R* value with a data-only red-noise uncertainty because the sampled " + "Rp/R* posterior was pinned against a search bound and automatic Rp/R* " + "posterior expansion was disabled. Tmid, a/Rs, inclination, and impact " + "parameter remain fitted parameters; their primary uncertainties apply the " + "same residual time-binning beta factor to the posterior uncertainties. " + "The flux baseline red-noise uncertainty is the out-of-transit baseline " + "component used for the final-plot baseline band." + ) + else: + params["Uncertainty interpretation note"] = ( + "The primary Rp/R* and radius-ratio area-depth uncertainties use model+red-noise " + "when available. The red-noise uncertainty inflates the data-fit standard error " + "by a residual time-binning beta factor before combining it with the model " + "posterior. The flux baseline red-noise uncertainty is the out-of-transit " + "baseline component used for the final-plot baseline band; the transit-depth " + "red-noise uncertainty already includes both the in-transit and baseline terms. " + "The same residual time-binning beta factor is applied to the posterior " + "uncertainties for Tmid, a/Rs, inclination, and impact parameter when reporting " + "their primary model+red-noise uncertainties. " + "The flux-scatter equivalent is also shown as a diagnostic of the full residual " + "scatter around the model." + ) + return params + + +def build_aavso_photometry_metadata(photometry_info): + if not isinstance(photometry_info, dict): + return {} + + selected_source_indices = photometry_info.get('selected_source_indices') + selected_source_count = None + if selected_source_indices is not None: + try: + selected_source_count = int(np.asarray(selected_source_indices).size) + except (TypeError, ValueError): + selected_source_count = None + + selected_times = numeric_series_summary(photometry_info.get('selected_fit_good_times')) + return { + 'method': photometry_method_from_info(photometry_info), + 'selected_comparison_star': photometry_info.get('comp_star_num'), + 'selected_comparison_coordinates': photometry_info.get('comp_star_coords'), + 'noise_budget_summary': photometry_info.get('noise_budget_summary'), + 'noise_budget_terms': photometry_info.get('noise_budget_terms'), + 'comparison_selection_basis': photometry_info.get('selection_basis'), + 'comparison_selection_metric': photometry_info.get('selection_metric'), + 'comparison_field_score': photometry_info.get('calibration_field_score'), + 'comparison_field_score_percent': ( + 100.0 * finite_float(photometry_info.get('calibration_field_score')) + if np.isfinite(finite_float(photometry_info.get('calibration_field_score'))) + else np.nan + ), + 'selected_comparison_ktmf': photometry_info.get('comparison_ktmf_metric'), + 'selected_comparison_eebls_snr': photometry_info.get('comparison_eebls_snr'), + 'selected_comparison_transit_delta_bic': photometry_info.get('comparison_transit_delta_bic'), + 'reused_selected_full_reduction_fit': photometry_info.get('reuse_selected_full_reduction_fit'), + 'selected_source_point_count': selected_source_count, + 'selected_fit_time_range': selected_times, + } + + +def build_aavso_aperture_metadata(photometry_info): + if not isinstance(photometry_info, dict): + return {} + + adaptive_summary = photometry_info.get('adaptive_summary') + payload = { + 'method': photometry_method_from_info(photometry_info), + 'aperture_index': photometry_info.get('aperture_index'), + 'annulus_index': photometry_info.get('annulus_index'), + 'configured_aperture_px': photometry_info.get('min_aperture'), + 'configured_annulus_px': photometry_info.get('min_annulus'), + 'adaptive': adaptive_summary is not None, + } + if not isinstance(adaptive_summary, dict): + return payload + + payload.update({ + 'aperture_sigma': adaptive_summary.get('aperture_sigma'), + 'annulus_sigma': adaptive_summary.get('annulus_sigma'), + 'aperture_px': { + 'median': adaptive_summary.get('aperture_median'), + 'std': adaptive_summary.get('aperture_std'), + 'min': adaptive_summary.get('aperture_min'), + 'max': adaptive_summary.get('aperture_max'), + }, + 'annulus_px': { + 'median': adaptive_summary.get('annulus_median'), + 'std': adaptive_summary.get('annulus_std'), + 'min': adaptive_summary.get('annulus_min'), + 'max': adaptive_summary.get('annulus_max'), + }, + 'fwhm_px': numeric_series_summary(adaptive_summary.get('fwhm_series')), + 'frame_sigma_px': numeric_series_summary(adaptive_summary.get('frame_sigma')), + 'sky_inner_px': numeric_series_summary(adaptive_summary.get('sky_inner_series')), + 'sky_outer_px': numeric_series_summary(adaptive_summary.get('sky_outer_series')), + 'sky_pixels': numeric_series_summary(adaptive_summary.get('sky_pixel_series')), + }) + return payload + + +def build_aavso_frame_filtering_metadata(fit, frame_filtering_info): + payload = dict(frame_filtering_info or {}) + + for source_key, target_key in ( + ('dropped_missing_wcs_files', 'missing_wcs_rejections'), + ('dropped_target_wcs_files', 'target_wcs_rejections'), + ('dropped_pointing_files', 'pointing_rejections'), + ): + if source_key in payload: + payload[target_key] = file_list_summary(payload.pop(source_key)) + + diagnostics = getattr(fit, 'frame_filter_diagnostics', None) + if diagnostics: + payload['lightcurve_filter_diagnostics'] = diagnostics + payload['lightcurve_dropped_point_count'] = sum( + int((diagnostic or {}).get('dropped_point_count', 0)) + for diagnostic in diagnostics + ) + final_residual_rejection = getattr(fit, 'final_residual_rejection', None) + if isinstance(final_residual_rejection, dict) and final_residual_rejection: + payload['final_residual_rejection'] = aavso_json_safe(final_residual_rejection) + return payload + + +def build_aavso_astrometry_metadata(astrometry_info, comp_star): + payload = dict(astrometry_info or {}) + if payload.get('wcs_file'): + payload['wcs_file'] = path_name(payload['wcs_file']) + if comp_star: + payload['comparison_star_aavso_header'] = comp_star + return payload + + +def build_aavso_bad_pixel_metadata(bad_pixel_info): + if not isinstance(bad_pixel_info, dict): + return {} + + payload = dict(bad_pixel_info) + for key in ('counts_path', 'mask_path'): + if payload.get(key): + payload[key] = path_name(payload[key]) + return payload + + +def format_parameter_with_error(value, error): + value = finite_float(value) + error = finite_float(error) + if not np.isfinite(value): + return None + if np.isfinite(error) and error >= 0: + return format_value_with_uncertainty(value, error) + return f"{round_to_2(value)} +/- n/a" + + +def format_percent_parameter_with_error(value, error): + text = format_parameter_with_error(value, error) + return f"{text} [%]" if text is not None else None + + +def formatted_transit_depth_parameters(fit, planet_dict=None, limb_darkening=None, empirical_uncertainty=None): + prior_parameters = planet_dict_transit_parameters( + planet_dict, + limb_darkening=limb_darkening, + fallback=getattr(fit, 'prior', None), + ) + prior_errors = planet_dict_transit_errors(planet_dict, limb_darkening=limb_darkening) + summary = fit_transit_depth_summary( + fit, + prior_parameters=prior_parameters, + prior_errors=prior_errors, + ) + + entries = {} + for label, value_key, error_key in ( + (AREA_DEPTH_LABEL, 'area_depth', 'area_depth_error'), + (OBSERVABLE_DEPTH_LABEL, 'observable_depth', 'observable_depth_error'), + (PRIOR_OBSERVABLE_DEPTH_LABEL, 'prior_observable_depth', 'prior_observable_depth_error'), + (OBSERVABLE_DEPTH_DELTA_LABEL, 'observable_depth_prior_delta', 'observable_depth_prior_delta_error'), + ): + text = format_percent_parameter_with_error(summary.get(value_key), summary.get(error_key)) + if text is not None: + entries[label] = text + + empirical_uncertainty = empirical_uncertainty or {} + if empirical_uncertainty.get('available'): + area_depth = finite_float(summary.get('area_depth')) + rprs = finite_float(empirical_uncertainty.get('rprs')) + rprs_prior_fallback = bool(empirical_uncertainty.get('rprs_prior_fallback_applied')) + + def area_error_percent_from_rprs_error(error): + error = finite_float(error) + if np.isfinite(area_depth) and np.isfinite(rprs) and np.isfinite(error) and error >= 0: + return float(200.0 * abs(rprs) * error) + return np.nan + + model_area_error = area_error_percent_from_rprs_error( + empirical_uncertainty.get('model_rprs_uncertainty') + ) + data_area_error = area_error_percent_from_rprs_error( + empirical_uncertainty.get('data_rprs_uncertainty') + ) + combined_area_error = area_error_percent_from_rprs_error( + empirical_uncertainty.get('combined_rprs_uncertainty') + ) + flux_scatter_area_error = area_error_percent_from_rprs_error( + empirical_uncertainty.get('data_rprs_flux_scatter_uncertainty') + ) + if np.isfinite(area_depth) and np.isfinite(combined_area_error): + combined_text = format_percent_parameter_with_error(area_depth, combined_area_error) + if combined_text is not None: + entries[AREA_DEPTH_LABEL] = combined_text + if rprs_prior_fallback: + entries[f"{AREA_DEPTH_LABEL} prior-assumed data-only uncertainty"] = combined_text + else: + entries[f"{AREA_DEPTH_LABEL} model+red-noise uncertainty"] = combined_text + if not rprs_prior_fallback and np.isfinite(area_depth) and np.isfinite(model_area_error): + text = format_percent_parameter_with_error(area_depth, model_area_error) + if text is not None: + entries[f"{AREA_DEPTH_LABEL} model-fit uncertainty"] = text + if np.isfinite(area_depth) and np.isfinite(data_area_error): + text = format_percent_parameter_with_error(area_depth, data_area_error) + if text is not None: + if rprs_prior_fallback: + entries[f"{AREA_DEPTH_LABEL} prior-assumed data-fit uncertainty"] = text + else: + entries[f"{AREA_DEPTH_LABEL} data-fit red-noise uncertainty"] = text + if np.isfinite(area_depth) and np.isfinite(flux_scatter_area_error): + text = format_percent_parameter_with_error(area_depth, flux_scatter_area_error) + if text is not None: + entries[f"{AREA_DEPTH_LABEL} flux-scatter equivalent"] = text + return entries + + +def empirical_red_noise_error_scale(empirical_uncertainty): + empirical_uncertainty = empirical_uncertainty or {} + if not empirical_uncertainty.get('available'): + return 1.0 + + beta = finite_float(empirical_uncertainty.get('red_noise_beta_factor')) + if np.isfinite(beta) and beta > 1.0: + return float(beta) + return 1.0 + + +def fit_parameter_model_data_uncertainty(fit, parameter_name, empirical_uncertainty=None): + errors = getattr(fit, 'errors', {}) or {} + model_error = finite_float(errors.get(parameter_name)) + if not np.isfinite(model_error) or model_error < 0: + return np.nan + + return float(model_error * empirical_red_noise_error_scale(empirical_uncertainty)) + + +def fit_rprs_report_error(fit, empirical_uncertainty=None): + if empirical_uncertainty is None: + empirical_uncertainty = fit_empirical_transit_uncertainty(fit) + if not isinstance(empirical_uncertainty, dict): + empirical_uncertainty = {} + + report_error = finite_float(empirical_uncertainty.get('combined_rprs_uncertainty')) + if np.isfinite(report_error) and report_error >= 0: + return report_error + + errors = getattr(fit, 'errors', {}) or {} + report_error = finite_float(errors.get('rprs')) + if np.isfinite(report_error) and report_error >= 0: + return report_error + + report_error = finite_float(getattr(fit, 'rprs_prior_fallback_data_uncertainty', np.nan)) + if np.isfinite(report_error) and report_error >= 0: + return report_error + + return np.nan + + +def fit_impact_parameter_value_error(fit, errors_override=None): + parameters = getattr(fit, 'parameters', {}) or {} + errors = getattr(fit, 'errors', {}) or {} + errors_override = errors_override or {} + sample_parameters = getattr(fit, 'sample_parameters', {}) or {} + sample_errors = getattr(fit, 'sample_errors', {}) or {} + + if 'b' in sample_parameters: + impact_parameter = finite_float(sample_parameters.get('b')) + impact_error = finite_float(errors_override.get('b'), finite_float(sample_errors.get('b'))) + if np.isfinite(impact_parameter): + return impact_parameter, impact_error + + if 'b' in parameters: + impact_parameter = finite_float(parameters.get('b')) + impact_error = finite_float(errors_override.get('b'), finite_float(errors.get('b'))) + if np.isfinite(impact_parameter): + return impact_parameter, impact_error + + ars = finite_float(parameters.get('ars')) + inc = finite_float(parameters.get('inc')) + if not np.isfinite(ars) or not np.isfinite(inc): + return np.nan, np.nan + + ecc = finite_float(parameters.get('ecc'), 0.0) + omega = np.deg2rad(finite_float(parameters.get('omega'), 0.0)) + denominator = 1.0 + ecc * np.sin(omega) + if not np.isfinite(denominator) or np.isclose(denominator, 0.0): + return np.nan, np.nan + + scale_factor = (1.0 - ecc ** 2) / denominator + inc_rad = np.deg2rad(inc) + impact_parameter = scale_factor * ars * np.cos(inc_rad) + + ars_error = finite_float(errors_override.get('ars'), finite_float(errors.get('ars'))) + inc_error = finite_float(errors_override.get('inc'), finite_float(errors.get('inc'))) + if np.isfinite(ars_error) and np.isfinite(inc_error): + impact_error = np.hypot( + scale_factor * np.cos(inc_rad) * ars_error, + scale_factor * ars * np.sin(inc_rad) * np.deg2rad(inc_error), + ) + else: + impact_error = np.nan + + return float(impact_parameter), float(impact_error) if np.isfinite(impact_error) else np.nan class OutputFiles: @@ -25,61 +2530,740 @@ def __init__(self, fit, p_dict, i_dict, durs): self.dir = Path(self.i_dict['save']) def final_lightcurve(self, phase): - params_file = self.dir / "temp" / f"FinalLightCurve_{self.p_dict['pName']}_{self.i_dict['date']}.csv" + params_file = self.dir / "working_artifacts" / safe_output_filename( + "FinalLightCurve", + self.p_dict['pName'], + filename_date_token(self.i_dict['date']), + extension="csv", + ) + + if getattr(self.fit, 'stellar_variability_only', False): + vsp_params = getattr(self.fit, 'stellar_variability_params', None) or [] + with params_file.open('w') as f: + target_name = self.p_dict.get('sName', self.p_dict['pName']) + f.write(f"# FINAL STELLAR VARIABILITY TIMESERIES OF {target_name}\n") + reference_label = ( + getattr(self.fit, 'stellar_variability_reference_label', None) + or (vsp_params[0].get('cname') if vsp_params else None) + or 'selected comparison reference' + ) + f.write(f"# DIFFERENTIAL_MAGNITUDE_REFERENCE={reference_label}\n") + f.write("# DIFFERENTIAL_MAGNITUDE_AIRMASS_CORRECTED=NO\n") + f.write( + "# BJD_TDB,Apparent Magnitude,Apparent Magnitude Uncertainty," + "Raw Differential Magnitude,Raw Differential Magnitude Uncertainty,Band,Airmass\n" + ) + for vsp_p in vsp_params: + time_value = finite_float(vsp_p.get('time')) + mag_value = format_magnitude(vsp_p.get('mag'), default=None) + mag_error = format_magnitude_error(vsp_p.get('mag_err'), default=None) + if not np.isfinite(time_value) or mag_value is None or mag_error is None: + continue + differential_mag, differential_error = differential_magnitude_from_vsp_param( + vsp_p + ) + differential_mag_text = format_magnitude( + differential_mag, + default="na", + digits=MAGNITUDE_DECIMAL_PLACES, + ) + differential_error_text = format_magnitude_error( + differential_error, + default="na", + digits=MAGNITUDE_DECIMAL_PLACES, + ) + band = vsp_p.get('mag_band') or self.i_dict.get('filter') or 'V' + airmass = finite_float(vsp_p.get('airmass')) + airmass_text = f"{airmass}" if np.isfinite(airmass) else "na" + f.write( + f"{time_value}, {mag_value}, {mag_error}, {differential_mag_text}, " + f"{differential_error_text}, {band}, {airmass_text}\n" + ) + return + + magnitude_series = magnitude_series_from_fit(self.fit) + band = magnitude_series['band'] or self.i_dict.get('filter') or 'na' + fit_times = np.asarray(getattr(self.fit, 'time', []), dtype=float).reshape(-1) + detrend_model = np.asarray(aavso_detrend_model(self.fit), dtype=float).reshape(-1) + if detrend_model.shape != fit_times.shape: + detrend_model = np.ones(fit_times.shape, dtype=float) + airmass_values = np.asarray( + getattr(self.fit, 'airmass', np.full(fit_times.shape, np.nan)), + dtype=float, + ).reshape(-1) + if airmass_values.shape != fit_times.shape: + airmass_values = np.full(fit_times.shape, np.nan, dtype=float) + corrected_flux_error = np.asarray( + getattr(self.fit, 'detrendederr', []), + dtype=float, + ).reshape(-1) + if corrected_flux_error.shape != fit_times.shape: + fit_data_error = np.asarray(getattr(self.fit, 'dataerr', []), dtype=float).reshape(-1) + if fit_data_error.shape == fit_times.shape: + corrected_flux_error = np.divide( + fit_data_error, + detrend_model, + out=np.full(fit_times.shape, np.nan, dtype=float), + where=np.isfinite(detrend_model) & (detrend_model > 0), + ) + else: + corrected_flux_error = np.full(fit_times.shape, np.nan, dtype=float) with params_file.open('w') as f: f.write(f"# FINAL TIMESERIES OF {self.p_dict['pName']}\n") - f.write("# BJD_TDB,Orbital Phase,Flux,Uncertainty,Model,Airmass\n") + reference_label = ( + getattr(self.fit, 'differential_magnitude_reference_label', None) + or getattr(self.fit, 'stellar_variability_reference_label', None) + or 'selected comparison reference' + ) + f.write(f"# DIFFERENTIAL_MAGNITUDE_REFERENCE={reference_label}\n") + f.write( + "# DIFFERENTIAL_MAGNITUDE_AIRMASS_CORRECTED=" + f"{'YES' if magnitude_series['airmass_corrected'] else 'NO'}\n" + ) + f.write( + "# DIFFERENTIAL_MAGNITUDE_CORRECTION=" + f"{magnitude_series['correction_type']}\n" + ) + f.write( + "# BJD_TDB,Orbital Phase,Flux,Uncertainty,Model,Airmass,Detrend Correction Function," + "Raw Differential Magnitude,Raw Differential Magnitude Uncertainty," + "Corrected Differential Magnitude,Corrected Differential Magnitude Uncertainty," + "Apparent Magnitude,Apparent Magnitude Uncertainty,Band\n" + ) + + for row_index, (bjd, phase, flux, fluxerr, model, am, correction_value) in enumerate(zip( + self.fit.time, + phase, + self.fit.detrended, + corrected_flux_error, + self.fit.transit, + airmass_values, + detrend_model)): + airmass_text = str(am) if np.isfinite(am) else 'na' + correction_text = str(correction_value) if np.isfinite(correction_value) else 'na' + row = ( + f"{bjd}, {phase}, {flux}, {fluxerr}, {model}, " + f"{airmass_text}, {correction_text}" + ) + raw_differential_mag_text = format_magnitude( + magnitude_series['raw_differential_magnitude'][row_index], + default="na", + digits=MAGNITUDE_DECIMAL_PLACES, + ) + raw_differential_error_text = format_magnitude_error( + magnitude_series['raw_differential_magnitude_error'][row_index], + default="na", + digits=MAGNITUDE_DECIMAL_PLACES, + ) + corrected_differential_mag_text = format_magnitude( + magnitude_series['corrected_differential_magnitude'][row_index], + default="na", + digits=MAGNITUDE_DECIMAL_PLACES, + ) + corrected_differential_error_text = format_magnitude_error( + magnitude_series['corrected_differential_magnitude_error'][row_index], + default="na", + digits=MAGNITUDE_DECIMAL_PLACES, + ) + apparent_mag_text = format_magnitude( + magnitude_series['apparent_magnitude'][row_index], + default="na", + ) + apparent_error_text = format_magnitude_error( + magnitude_series['apparent_magnitude_error'][row_index], + default="na", + ) + row = ( + f"{row}, {raw_differential_mag_text}, {raw_differential_error_text}, " + f"{corrected_differential_mag_text}, {corrected_differential_error_text}, " + f"{apparent_mag_text}, {apparent_error_text}, {band}" + ) + f.write(f"{row}\n") + + def differential_magnitude(self): + target_name = self.p_dict.get('sName') or self.p_dict.get('pName') or 'target' + return write_differential_magnitude_csv( + self.fit, + self.dir, + target_name, + observation_date=self.i_dict.get('date'), + observed_filter=( + self.i_dict.get('observed_filter') + or self.i_dict.get('filter') + ), + out_of_transit_only=False, + ) + + def stellar_variability_differential_magnitude(self): + """Write the raw, out-of-transit stellar-variability counterpart.""" + if getattr(self.fit, 'stellar_variability_only', False): + return None + fit_shape = np.asarray(getattr(self.fit, 'data', []), dtype=float).shape + target_shape = np.asarray( + getattr(self.fit, 'stellar_variability_target_flux', []), + dtype=float, + ).shape + reference_shape = np.asarray( + getattr(self.fit, 'stellar_variability_comp_flux', []), + dtype=float, + ).shape + if target_shape != fit_shape or reference_shape != fit_shape: + return None + target_name = self.p_dict.get('sName') or self.p_dict.get('pName') or 'target' + return write_differential_magnitude_csv( + self.fit, + self.dir, + target_name, + observation_date=self.i_dict.get('date'), + observed_filter=( + self.i_dict.get('observed_filter') + or self.i_dict.get('filter') + ), + out_of_transit_only=True, + apply_airmass_correction=False, + filename_prefix='StellarVariabilityDifferentialMagnitude', + ) + + def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coords=None, min_aper=None, + min_annul=None, adaptive_summary=None, photometry_info=None, + publish_to_root=False): + params_file = self.dir / "working_artifacts" / safe_output_filename( + "FinalParams", + self.p_dict['pName'], + filename_date_token(self.i_dict['date']), + extension="json", + ) + + if getattr(self.fit, 'stellar_variability_only', False): + exclusion = getattr(self.fit, 'stellar_variability_transit_exclusion', {}) or {} + scatter = getattr(self.fit, 'stellar_variability_scatter', np.nan) + params_num = { + "Analysis Mode": "Stellar variability only", + "Transit model fitting": "Skipped", + "Out-of-transit lightcurve point count": str(len(getattr(self.fit, 'time', []))), + "Predicted in-transit points excluded": str(exclusion.get('rejected_point_count', 0)), + } + duration = exclusion.get('duration_days', np.nan) + if np.isfinite(duration): + params_num["Excluded transit-window duration (day)"] = f"{duration:.8f}" + if np.isfinite(scatter): + params_num["Residual scatter around flat stellar-variability model"] = f"{scatter * 100.0:.4f} %" + note = exclusion.get('note') + if note: + params_num["Transit-window exclusion note"] = str(note) + if getattr(self.fit, 'airmass_fit_skipped', False): + params_num["Airmass correction"] = getattr( + self.fit, + 'airmass_correction_note', + "Skipped; no airmass correction applied.", + ) + if isinstance(photometry_info, dict) and photometry_info.get('noise_budget_summary'): + params_num["Photometry noise budget"] = str(photometry_info.get('noise_budget_summary')) + if photometry_info.get('noise_budget_terms'): + params_num["Photometry noise budget terms"] = ", ".join( + str(term) for term in photometry_info.get('noise_budget_terms') + ) + + if vsp_params: + params_num["Variable Reference Star"] = stellar_variability_reference_summary(vsp_params[0]) + if vsp_params[0].get('ensemble_reference'): + params_num["Variable Reference Measurement"] = ( + f"Combined {len(vsp_params)} out-of-transit target measurements against the " + "calibrated comparison-star ensemble; AID rows list the BJD_TDB timestamps used." + ) + else: + params_num["Variable Reference Measurement"] = ( + f"Remeasured {len(vsp_params)} out-of-transit target/reference point(s) " + "against the stellar-variability reference catalog star; AID rows list the " + "BJD_TDB timestamps used." + ) + + if phot_opt: + if comp_star == 'ensemble': + reference_text = "ensemble" + else: + reference_text = ( + f"#{comp_star} - {comp_coords}" + if comp_star is not None and min_aper is not None and min_aper >= 0 + else str(comp_star) + ) + params_num["Stellar Variability Reference Star"] = reference_text + if min_aper == 0: + params_num["Optimal Method"] = "PSF photometry" + else: + if adaptive_summary: + params_num["Adaptive Aperture Scale"] = f"{adaptive_summary['aperture_sigma']:.2f} sigma" + params_num["Adaptive Annulus Scale"] = f"{adaptive_summary['annulus_sigma']:.2f} sigma" + params_num["Optimal Aperture"] = ( + f"{adaptive_summary['aperture_median']:.2f} +/- " + f"{adaptive_summary['aperture_std']:.2f} px" + ) + params_num["Aperture Range"] = ( + f"{adaptive_summary['aperture_min']:.2f} to " + f"{adaptive_summary['aperture_max']:.2f} px" + ) + params_num["Optimal Annulus"] = ( + f"{adaptive_summary['annulus_median']:.2f} +/- " + f"{adaptive_summary['annulus_std']:.2f} px" + ) + params_num["Annulus Range"] = ( + f"{adaptive_summary['annulus_min']:.2f} to " + f"{adaptive_summary['annulus_max']:.2f} px" + ) + else: + params_num["Optimal Aperture"] = f"{abs(min_aper)}" + params_num["Optimal Annulus"] = f"{min_annul}" - for bjd, phase, flux, fluxerr, model, am in zip(self.fit.time, phase, self.fit.detrended, - self.fit.dataerr / self.fit.airmass_model, - self.fit.transit, self.fit.airmass_model): - f.write(f"{bjd}, {phase}, {flux}, {fluxerr}, {model}, {am}\n") + final_params = {'FINAL STELLAR VARIABILITY PARAMETERS': params_num} + with params_file.open('w') as f: + dump(final_params, f, indent=4) + if publish_to_root: + root_params_file = self.dir / params_file.name + if root_params_file != params_file: + root_params_file.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(params_file, root_params_file) + return - def final_planetary_params(self, phot_opt, vsp_params, comp_star=None, comp_coords=None, min_aper=None, min_annul=None): - params_file = self.dir / "temp" / f"FinalParams_{self.p_dict['pName']}_{self.i_dict['date']}.json" + transit_qc = getattr(self.fit, 'transit_qc', None) + fit_quality = build_fit_quality_metadata(self.fit) + empirical_uncertainty = fit_empirical_transit_uncertainty(self.fit, fit_quality=fit_quality) + qc_residual_scatter = np.nan + if isinstance(transit_qc, dict): + qc_residual_scatter = transit_qc.get('residual_scatter', np.nan) + if not np.isfinite(qc_residual_scatter): + residuals = np.asarray(getattr(self.fit, 'residuals', np.array([])), dtype=float) + data = np.asarray(getattr(self.fit, 'data', np.array([])), dtype=float) + if residuals.size and data.size: + if residuals.shape == data.shape: + median_flux = np.nanmedian(data) + if np.isfinite(median_flux) and median_flux != 0: + qc_residual_scatter = float(np.std(residuals) / median_flux) + elif residuals.size == 1: + median_flux = np.nanmedian(data) + if np.isfinite(median_flux) and median_flux != 0: + qc_residual_scatter = float(abs(residuals.reshape(-1)[0]) / median_flux) + headline_params = format_transit_qc_headline_final_params(transit_qc) + rprs_report_error = fit_rprs_report_error( + self.fit, + empirical_uncertainty=empirical_uncertainty, + ) + if not np.isfinite(rprs_report_error) or rprs_report_error < 0: + rprs_report_error = finite_float(self.p_dict.get('rprsUnc')) + tmid_report_error = fit_parameter_model_data_uncertainty( + self.fit, + 'tmid', + empirical_uncertainty=empirical_uncertainty, + ) + if not np.isfinite(tmid_report_error) or tmid_report_error < 0: + tmid_report_error = self.fit.errors['tmid'] + inc_report_error = fit_parameter_model_data_uncertainty( + self.fit, + 'inc', + empirical_uncertainty=empirical_uncertainty, + ) + if not np.isfinite(inc_report_error) or inc_report_error < 0: + inc_report_error = self.fit.errors['inc'] + ars_report_error = fit_parameter_model_data_uncertainty( + self.fit, + 'ars', + empirical_uncertainty=empirical_uncertainty, + ) + if not np.isfinite(ars_report_error) or ars_report_error < 0: + ars_report_error = self.fit.errors.get('ars', np.nan) + core_params = { + "Mid-Transit Time (Tmid)": ( + f"{format_value_with_uncertainty(self.fit.parameters['tmid'], tmid_report_error)} BJD_TDB" + ), + "Ratio of Planet to Stellar Radius (Rp/R*)": format_value_with_uncertainty( + self.fit.parameters['rprs'], rprs_report_error + ), + "Orbital Inclination (inc)": ( + f"{format_value_with_uncertainty(self.fit.parameters['inc'], inc_report_error)} " + ), + } + depth_params = formatted_transit_depth_parameters( + self.fit, + self.p_dict, + empirical_uncertainty=empirical_uncertainty, + ) params_num = { - "Mid-Transit Time (Tmid)": f"{round_to_2(self.fit.parameters['tmid'], self.fit.errors['tmid'])} +/- " - f"{round_to_2(self.fit.errors['tmid'])} BJD_TDB", - "Ratio of Planet to Stellar Radius (Rp/R*)": f"{round_to_2(self.fit.parameters['rprs'], self.fit.errors['rprs'])} +/- " - f"{round_to_2(self.fit.errors['rprs'])}", - "Transit depth (Rp/Rs)^2": f"{round_to_2(100. * (self.fit.parameters['rprs'] ** 2.))} +/- " - f"{round_to_2(100. * 2. * self.fit.parameters['rprs'] * self.fit.errors['rprs'])} [%]", - "Orbital Inclination (inc)": f"{round_to_2(self.fit.parameters['inc'], self.fit.errors['inc'])} +/- " - f"{round_to_2(self.fit.errors['inc'])} ", - "Airmass coefficient 1 (a1)": f"{round_to_2(self.fit.parameters['a1'], self.fit.errors['a1'])} +/- " - f"{round_to_2(self.fit.errors['a1'])}", - "Airmass coefficient 2 (a2)": f"{round_to_2(self.fit.parameters['a2'], self.fit.errors['a2'])} +/- " - f"{round_to_2(self.fit.errors['a2'])}", - "Scatter in the residuals of the lightcurve fit is": f"{round_to_2(100. * std(self.fit.residuals / median(self.fit.data)))} %", + **headline_params, + "Mid-Transit Time (Tmid)": core_params["Mid-Transit Time (Tmid)"], + "Ratio of Planet to Stellar Radius (Rp/R*)": core_params["Ratio of Planet to Stellar Radius (Rp/R*)"], + **depth_params, + "Orbital Inclination (inc)": core_params["Orbital Inclination (inc)"], + } + ars_text = format_parameter_with_error( + self.fit.parameters.get('ars'), + ars_report_error, + ) + if ars_text is not None: + params_num["Ratio of Distance to Stellar Radius (a/Rs)"] = ars_text + model_impact_parameter, model_impact_error = fit_impact_parameter_value_error(self.fit) + impact_error_overrides = { + 'ars': ars_report_error, + 'inc': inc_report_error, } + if ( + np.isfinite(model_impact_error) + and ('b' in (getattr(self.fit, 'parameters', {}) or {}) + or 'b' in (getattr(self.fit, 'sample_parameters', {}) or {})) + ): + impact_error_overrides['b'] = model_impact_error * empirical_red_noise_error_scale( + empirical_uncertainty + ) + impact_parameter, impact_error = fit_impact_parameter_value_error( + self.fit, + errors_override=impact_error_overrides, + ) + impact_text = format_parameter_with_error(impact_parameter, impact_error) + if impact_text is not None: + params_num["Impact Parameter (b)"] = impact_text + if empirical_uncertainty.get('available'): + tmid_model_text = ( + f"{format_value_with_uncertainty(self.fit.parameters['tmid'], self.fit.errors['tmid'])} " + "BJD_TDB" + ) + tmid_combined_text = core_params["Mid-Transit Time (Tmid)"] + params_num["Mid-Transit Time (Tmid) model-fit uncertainty"] = tmid_model_text + params_num["Mid-Transit Time (Tmid) model+red-noise uncertainty"] = tmid_combined_text + + inc_model_text = ( + f"{format_value_with_uncertainty(self.fit.parameters['inc'], self.fit.errors['inc'])} " + ) + params_num["Orbital Inclination (inc) model-fit uncertainty"] = inc_model_text + params_num["Orbital Inclination (inc) model+red-noise uncertainty"] = core_params[ + "Orbital Inclination (inc)" + ] + + ars_model_text = format_parameter_with_error( + self.fit.parameters.get('ars'), + self.fit.errors.get('ars'), + ) + if ars_model_text is not None: + params_num["Ratio of Distance to Stellar Radius (a/Rs) model-fit uncertainty"] = ars_model_text + if ars_text is not None: + params_num["Ratio of Distance to Stellar Radius (a/Rs) model+red-noise uncertainty"] = ars_text + + impact_model_text = format_parameter_with_error(model_impact_parameter, model_impact_error) + if impact_model_text is not None: + params_num["Impact Parameter (b) model-fit uncertainty"] = impact_model_text + if impact_text is not None: + params_num["Impact Parameter (b) model+red-noise uncertainty"] = impact_text + if getattr(self.fit, 'ns_type', None) is not None: + params_num["Fit parameter point estimate"] = ( + "Best-fit likelihood point; uncertainties are posterior spread." + ) + prefit_refinement_note = getattr(self.fit, 'prefit_refinement_note', None) + if prefit_refinement_note: + params_num["Prefit refinement note"] = str(prefit_refinement_note) + geometry_prior_note = getattr(self.fit, 'partial_transit_geometry_prior_assumption_note', None) + if geometry_prior_note: + params_num["Prior-assumed partial-transit geometry note"] = str(geometry_prior_note) + ars_prior_fallback_note = getattr(self.fit, 'ars_prior_fallback_note', None) + if ars_prior_fallback_note: + params_num["a/Rs prior fallback note"] = str(ars_prior_fallback_note) + oot_baseline_parameter_note = getattr(self.fit, 'oot_baseline_parameter_fit_note', None) + if oot_baseline_parameter_note: + params_num["Out-of-transit baseline parameter-fit note"] = str(oot_baseline_parameter_note) + oot_baseline_note = getattr(self.fit, 'oot_baseline_detrending_note', None) + if oot_baseline_note: + params_num["Out-of-transit baseline detrending note"] = str(oot_baseline_note) + oot_baseline_metadata = out_of_transit_baseline_detrending_metadata(self.fit) + if oot_baseline_metadata.get('applied'): + params_num["Out-of-transit baseline detrending model"] = str( + oot_baseline_metadata['model'] + ) + for output_label, metadata_key in ( + ("Out-of-transit baseline reference time (BJD_TDB)", 'reference_time_bjd_tdb'), + ("Out-of-transit baseline intercept", 'intercept'), + ("Out-of-transit baseline slope (per day)", 'slope_per_day'), + ): + value = finite_float(oot_baseline_metadata.get(metadata_key)) + if np.isfinite(value): + params_num[output_label] = str(value) + sparse_posterior_note = getattr(self.fit, 'sparse_posterior_live_point_extension_note', None) + if sparse_posterior_note: + params_num["Sparse posterior live-point extension note"] = str(sparse_posterior_note) + final_residual_note = getattr(self.fit, 'final_residual_rejection_note', None) + if final_residual_note: + params_num["Final residual rejection note"] = str(final_residual_note) + if np.isfinite(qc_residual_scatter): + params_num["Residual scatter around full model fit"] = f"{qc_residual_scatter * 100.0:.4f} %" + params_num.update(format_fit_quality_final_params(fit_quality)) + params_num.update(format_empirical_transit_uncertainty_final_params(empirical_uncertainty)) + params_num.update(format_ktmf_decision_final_params(self.fit, photometry_info)) + if isinstance(photometry_info, dict) and photometry_info.get('noise_budget_summary'): + params_num["Photometry noise budget"] = str(photometry_info.get('noise_budget_summary')) + if photometry_info.get('noise_budget_terms'): + params_num["Photometry noise budget terms"] = ", ".join( + str(term) for term in photometry_info.get('noise_budget_terms') + ) + if getattr(self.fit, 'airmass_fit_skipped', False): + params_num["Airmass correction"] = getattr( + self.fit, + 'airmass_correction_note', + "Skipped; no airmass correction applied.", + ) + elif baseline_fixed_after_detrending(self.fit): + pre_detrending_report = pre_detrending_baseline_report(self.fit) + if pre_detrending_report is not None: + if pre_detrending_report.get('source'): + params_num["Pre-detrending baseline source"] = str( + pre_detrending_report['source'] + ) + if pre_detrending_report.get('scale_text'): + scale_parameter = pre_detrending_report['scale_parameter'] + scale_label = ( + "Pre-detrending baseline flux (a0)" + if scale_parameter == 'a0' + else "Pre-detrending airmass coefficient 1 (a1)" + ) + params_num[scale_label] = pre_detrending_report['scale_text'] + if pre_detrending_report.get('a2_text'): + params_num["Pre-detrending airmass coefficient 2 (a2)"] = ( + pre_detrending_report['a2_text'] + ) + if 'a0' in self.fit.parameters: + params_num["Baseline flux (a0)"] = fixed_detrended_baseline_text( + self.fit.parameters['a0'] + ) + else: + params_num["Flux normalization (a1)"] = fixed_detrended_baseline_text( + self.fit.parameters['a1'] + ) + if 'a2' in self.fit.parameters: + params_num["Airmass coefficient 2 (a2)"] = fixed_detrended_baseline_text( + self.fit.parameters['a2'] + ) + else: + if 'a0' in self.fit.parameters: + a0_error = self.fit.errors.get('a0') if isinstance(self.fit.errors, dict) else None + if a0_error is not None and np.isfinite(a0_error): + params_num["Baseline flux (a0)"] = format_value_with_uncertainty( + self.fit.parameters['a0'], a0_error + ) + else: + params_num["Baseline flux (a0)"] = ( + f"{round_to_2(self.fit.parameters['a0'])} (fixed; uncertainty unavailable)" + ) + else: + a1_error = self.fit.errors.get('a1') if isinstance(self.fit.errors, dict) else None + if a1_error is not None and np.isfinite(a1_error): + params_num["Flux normalization (a1)"] = format_value_with_uncertainty( + self.fit.parameters['a1'], a1_error + ) + else: + params_num["Flux normalization (a1)"] = ( + f"{round_to_2(self.fit.parameters['a1'])} (fixed; uncertainty unavailable)" + ) + if 'a2' in self.fit.parameters: + a2_error = self.fit.errors.get('a2') if isinstance(self.fit.errors, dict) else None + if a2_error is not None and np.isfinite(a2_error): + params_num["Airmass coefficient 2 (a2)"] = format_value_with_uncertainty( + self.fit.parameters['a2'], a2_error + ) + else: + params_num["Airmass coefficient 2 (a2)"] = ( + f"{round_to_2(self.fit.parameters['a2'])} (fixed; uncertainty unavailable)" + ) + + if isinstance(transit_qc, dict) and transit_qc: + qc_status = transit_qc.get('status') + qc_summary = transit_qc.get('summary') + qc_notes = transit_qc.get('notes') or [] + qc_delta_bic = transit_qc.get('delta_bic', np.nan) + qc_delta_chi2 = transit_qc.get('delta_chi2', np.nan) + qc_rprs_sigma = transit_qc.get('rprs_sigma', np.nan) + qc_duration_ratio = transit_qc.get('duration_ratio', np.nan) + qc_eebls_depth_snr = transit_qc.get('eebls_depth_snr', np.nan) + qc_deviation_metric = transit_qc.get('deviation_from_expected_value', np.nan) + qc_rprs_deviation_sigma = transit_qc.get('rprs_deviation_sigma', np.nan) + qc_rprs_deviation_fit_unc = transit_qc.get('rprs_deviation_fit_unc', np.nan) + qc_rprs_deviation_model_unc = transit_qc.get('rprs_deviation_model_fit_unc', np.nan) + qc_rprs_deviation_data_unc = transit_qc.get('rprs_deviation_data_fit_unc', np.nan) + qc_rprs_deviation_expected_unc = transit_qc.get('rprs_deviation_expected_unc', np.nan) + qc_rprs_deviation_comparison_unc = transit_qc.get('rprs_deviation_unc', np.nan) + qc_sigma_threshold = transit_qc.get('deviation_sigma_threshold', np.nan) + qc_ktmf = transit_qc.get('ktmf_metric', np.nan) + qc_ktmf_contributions = transit_qc.get('ktmf_contributions') or [] + + if qc_status: + params_num["Transit detection QC"] = str(qc_status).upper() + if qc_summary: + params_num["Transit vs flat model"] = qc_summary + if np.isfinite(qc_delta_bic): + params_num["Transit vs flat Delta BIC"] = f"{qc_delta_bic:.2f}" + if np.isfinite(qc_delta_chi2): + params_num["Transit vs flat Delta chi2"] = f"{qc_delta_chi2:.2f}" + if np.isfinite(qc_rprs_sigma): + params_num["Transit depth significance"] = f"{qc_rprs_sigma:.2f} sigma" + if np.isfinite(qc_duration_ratio): + params_num["Transit duration consistency"] = f"{qc_duration_ratio:.2f}x modeled duration" + if np.isfinite(qc_eebls_depth_snr): + params_num["EEBLS depth SNR"] = f"{qc_eebls_depth_snr:.2f}" + if np.isfinite(qc_deviation_metric): + params_num["Deviation From Expected Value"] = f"{qc_deviation_metric:.2f} / 1.00" + if np.isfinite(qc_sigma_threshold): + params_num["Expected-value QC threshold"] = f"{qc_sigma_threshold:.2f} sigma" + if np.isfinite(qc_rprs_deviation_sigma): + params_num["Expected-value Rp/R* deviation"] = f"{qc_rprs_deviation_sigma:.2f} sigma" + if np.isfinite(qc_rprs_deviation_fit_unc): + params_num["Expected-value Rp/R* fit uncertainty used"] = ( + f"+/- {format_uncertainty(qc_rprs_deviation_fit_unc)}" + ) + if np.isfinite(qc_rprs_deviation_model_unc): + params_num["Expected-value Rp/R* model-fit uncertainty"] = ( + f"+/- {format_uncertainty(qc_rprs_deviation_model_unc)}" + ) + if np.isfinite(qc_rprs_deviation_data_unc): + params_num["Expected-value Rp/R* data-fit red-noise uncertainty"] = ( + f"+/- {format_uncertainty(qc_rprs_deviation_data_unc)}" + ) + if np.isfinite(qc_rprs_deviation_expected_unc): + params_num["Expected-value Rp/R* prior uncertainty"] = ( + f"+/- {format_uncertainty(qc_rprs_deviation_expected_unc)}" + ) + if np.isfinite(qc_rprs_deviation_comparison_unc): + params_num["Expected-value Rp/R* total comparison uncertainty"] = ( + f"+/- {format_uncertainty(qc_rprs_deviation_comparison_unc)}" + ) + if np.isfinite(qc_ktmf): + params_num["KTMF"] = f"{qc_ktmf:.2f} / 5.00" + for contribution_index, contribution in enumerate(qc_ktmf_contributions, start=1): + label = contribution.get('label', f'Component {contribution_index}') + detail = contribution.get('detail') or 'n/a' + available = bool(contribution.get('available')) + points = float(contribution.get('points', 0.0) or 0.0) + max_points = float(contribution.get('max_points', 0.0) or 0.0) + score = contribution.get('score', np.nan) + if available and np.isfinite(score): + params_num[f"KTMF contribution {contribution_index}"] = ( + f"{label}: +{points:.2f}/{max_points:.2f} (score={score:.2f}; {detail})" + ) + else: + params_num[f"KTMF contribution {contribution_index}"] = ( + f"{label}: +0.00/0.00 (unavailable; {detail})" + ) + if qc_notes: + params_num["Transit QC notes"] = " ".join(str(note) for note in qc_notes) - if vsp_params: - params_num["Variable Reference Star"] = f"AAVSO Label: {vsp_params[0]['cname']}, " + \ - f"Position: {vsp_params[0]['pos']}" + if vsp_params and not (phot_opt and comp_star is None): + params_num["Variable Reference Star"] = stellar_variability_reference_summary(vsp_params[0]) + measurement_summary = stellar_variability_measurement_summary(vsp_params, comp_star) + if measurement_summary: + params_num["Variable Reference Measurement"] = measurement_summary if phot_opt: - phot_ext = {"Best Comparison Star": f"#{comp_star} - {comp_coords}" if min_aper >= 0 else str(comp_star)} + if comp_star == 'ensemble': + transit_fit_comp_text = "ensemble" + else: + transit_fit_comp_text = ( + f"#{comp_star} - {comp_coords}" + if comp_star is not None and min_aper >= 0 + else str(comp_star) + ) + phot_ext = { + "Transit Fit Comparison Star": transit_fit_comp_text + } if min_aper == 0: phot_ext["Optimal Method"] = "PSF photometry" else: - phot_ext["Optimal Aperture"] = f"{abs(min_aper)}" - phot_ext["Optimal Annulus"] = f"{min_annul}" + if adaptive_summary: + phot_ext["Adaptive Aperture Scale"] = f"{adaptive_summary['aperture_sigma']:.2f} sigma" + phot_ext["Adaptive Annulus Scale"] = f"{adaptive_summary['annulus_sigma']:.2f} sigma" + phot_ext["Optimal Aperture"] = ( + f"{adaptive_summary['aperture_median']:.2f} +/- {adaptive_summary['aperture_std']:.2f} px" + ) + phot_ext["Aperture Range"] = ( + f"{adaptive_summary['aperture_min']:.2f} to {adaptive_summary['aperture_max']:.2f} px" + ) + phot_ext["Optimal Annulus"] = ( + f"{adaptive_summary['annulus_median']:.2f} +/- {adaptive_summary['annulus_std']:.2f} px" + ) + phot_ext["Annulus Range"] = ( + f"{adaptive_summary['annulus_min']:.2f} to {adaptive_summary['annulus_max']:.2f} px" + ) + else: + phot_ext["Optimal Aperture"] = f"{abs(min_aper)}" + phot_ext["Optimal Annulus"] = f"{min_annul}" params_num.update(phot_ext) - params_num["Transit Duration (day)"] = (f"{round_to_2(mean(self.durs), std(self.durs))} +/- " - f"{round_to_2(std(self.durs))}") + params_num["Transit Duration (day)"] = format_value_with_uncertainty( + mean(self.durs), std(self.durs) + ) final_params = {'FINAL PLANETARY PARAMETERS': params_num} with params_file.open('w') as f: dump(final_params, f, indent=4) + if publish_to_root: + root_params_file = self.dir / params_file.name + if root_params_file != params_file: + root_params_file.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(params_file, root_params_file) - def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5): + def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5, + photometry_info=None, astrometry_info=None, frame_filtering_info=None, + bad_pixel_info=None): priors_dict, filter_dict, results_dict = aavso_dicts(self.p_dict, self.fit, self.i_dict, self.durs, ld0, ld1, ld2, ld3) + aavso_airmass_terms = aavso_airmass_results(self.fit) + detrend_model = aavso_detrend_model(self.fit) + undetrended_flux, undetrended_flux_error = aavso_undetrended_flux_series( + self.fit, + detrend_model, + ) + qc_metadata = build_aavso_qc_metadata(self.fit) + fit_quality_metadata = build_fit_quality_metadata(self.fit) + rprs_report_error = fit_rprs_report_error(self.fit) + if not np.isfinite(rprs_report_error) or rprs_report_error < 0: + rprs_report_error = finite_float(self.p_dict.get('rprsUnc')) + ktmf_decision_metadata = build_ktmf_decision_metadata(self.fit, photometry_info) + photometry_metadata = build_aavso_photometry_metadata(photometry_info) + aperture_metadata = build_aavso_aperture_metadata(photometry_info) + frame_filtering_metadata = build_aavso_frame_filtering_metadata(self.fit, frame_filtering_info) + astrometry_metadata = build_aavso_astrometry_metadata(astrometry_info, comp_star) + bad_pixel_metadata = build_aavso_bad_pixel_metadata(bad_pixel_info) + magnitude_series = magnitude_series_from_fit(self.fit) + obs_name = format_aavso_header_value(self.i_dict.get('obs_name')) + obs_name_header = f"#OBSNAME={obs_name}\n" if obs_name else "" + gaia_dist = format_aavso_header_value(self.p_dict.get('dist')) + gaia_pmra = format_aavso_header_value(self.p_dict.get('pm_ra')) + gaia_pmdec = format_aavso_header_value(self.p_dict.get('pm_dec')) + gaia_dist_header = f"#GAIADIST={gaia_dist}\n" if gaia_dist else "" + gaia_pmra_header = f"#GAIAPMRA={gaia_pmra}\n" if gaia_pmra else "" + gaia_pmdec_header = f"#GAIAPMDEC={gaia_pmdec}\n" if gaia_pmdec else "" + prior_period_text = format_value_with_uncertainty( + self.p_dict['pPer'], self.p_dict['pPerUnc'] + ) + prior_rprs_text = format_value_with_uncertainty( + self.p_dict['rprs'], self.p_dict['rprsUnc'] + ) + prior_ars_text = format_value_with_uncertainty( + self.p_dict['aRs'], self.p_dict['aRsUnc'] + ) + prior_inc_text = format_value_with_uncertainty( + self.p_dict['inc'], self.p_dict['incUnc'] + ) + prior_ld_texts = [format_value_with_uncertainty(*ld) for ld in (ld0, ld1, ld2, ld3)] + result_tmid_text = format_value_with_uncertainty( + self.fit.parameters['tmid'], self.fit.errors['tmid'] + ) + result_rprs_text = format_value_with_uncertainty( + self.fit.parameters['rprs'], rprs_report_error + ) + result_inc_text = format_value_with_uncertainty( + self.fit.parameters['inc'], self.fit.errors['inc'] + ) - params_file = self.dir / f"AAVSO_{self.p_dict['pName']}_{self.i_dict['date']}.txt" + params_file = aavso_output_directory(self.dir) / safe_output_filename( + "AAVSO", + self.p_dict['pName'], + filename_date_token(self.i_dict['date']), + extension="txt", + ) with params_file.open('w', encoding="utf-8") as f: f.write("#TYPE=EXOPLANET\n" # fixed @@ -88,33 +3272,139 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5): f"#SOFTWARE=EXOTIC v{__version__}\n" # fixed "#DELIM=,\n" # fixed "#DATE_TYPE=BJD_TDB\n" # fixed + f"#OBSDATE={format_aavso_header_value(self.i_dict.get('date'))}\n" + f"{obs_name_header}" f"#OBSTYPE={self.i_dict['camera']}\n" f"#STAR_NAME={self.p_dict['sName']}\n" # code yields - f"#EXOPLANET_NAME={self.p_dict['pName']}\n" # code yields + f"#EXOPLANET_NAME={format_aavso_exoplanet_name(self.p_dict['pName'])}\n" # code yields f"#BINNING={self.i_dict['pixel_bin']}\n" # user input f"#EXPOSURE_TIME={self.i_dict.get('exposure', -1)}\n" # UI + f"#OBSLAT={format_aavso_header_value(self.i_dict.get('lat'))}\n" + f"#OBSLON={format_aavso_header_value(self.i_dict.get('long'))}\n" + f"#OBSELEV={format_aavso_header_value(self.i_dict.get('elev'))}\n" + f"{gaia_dist_header}" + f"{gaia_pmra_header}" + f"{gaia_pmdec_header}" f"#COMP_STAR-XC={dumps(comp_star)}\n" f"#NOTES={self.i_dict['notes']}\n" "#DETREND_PARAMETERS=AIRMASS, AIRMASS CORRECTION FUNCTION\n" # fixed "#MEASUREMENT_TYPE=Rnflux\n" # fixed f"#FILTER={self.i_dict['filter']}\n" f"#FILTER-XC={dumps(filter_dict)}\n" - f"#PRIORS=Period={round_to_2(self.p_dict['pPer'], self.p_dict['pPerUnc'])} +/- {round_to_2(self.p_dict['pPerUnc'])}" - f",Rp/R*={round_to_2(self.p_dict['rprs'], self.p_dict['rprsUnc'])} +/- {round_to_2(self.p_dict['rprsUnc'])}" - f",a/R*={round_to_2(self.p_dict['aRs'], self.p_dict['aRsUnc'])} +/- {round_to_2(self.p_dict['aRsUnc'])}" - f",inc={round_to_2(self.p_dict['inc'], self.p_dict['incUnc'])} +/- {round_to_2(self.p_dict['incUnc'])}" + f"#PRIORS=Period={prior_period_text}" + f",Rp/R*={prior_rprs_text}" + f",a/R*={prior_ars_text}" + f",inc={prior_inc_text}" f",ecc={round_to_2(self.p_dict['ecc'])}" - f",u0={round_to_2(ld0[0], ld0[1])} +/- {round_to_2(ld0[1])}" - f",u1={round_to_2(ld1[0], ld1[1])} +/- {round_to_2(ld1[1])}" - f",u2={round_to_2(ld2[0], ld2[1])} +/- {round_to_2(ld2[1])}" - f",u3={round_to_2(ld3[0], ld3[1])} +/- {round_to_2(ld3[1])}\n" + f",u0={prior_ld_texts[0]}" + f",u1={prior_ld_texts[1]}" + f",u2={prior_ld_texts[2]}" + f",u3={prior_ld_texts[3]}\n" f"#PRIORS-XC={dumps(priors_dict)}\n" # code yields - f"#RESULTS=Tc={round_to_2(self.fit.parameters['tmid'], self.fit.errors['tmid'])} +/- {round_to_2(self.fit.errors['tmid'])}" - f",Rp/R*={round_to_2(self.fit.parameters['rprs'], self.fit.errors['rprs'])} +/- {round_to_2(self.fit.errors['rprs'])}" - f",inc={round_to_2(self.fit.parameters['inc'], self.fit.errors['inc'])} +/- {round_to_2(self.fit.errors['inc'])}" - f",Am1={round_to_2(self.fit.parameters['a1'], self.fit.errors['a1'])} +/- {round_to_2(self.fit.errors['a1'])}" - f",Am2={round_to_2(self.fit.parameters['a2'], self.fit.errors['a2'])} +/- {round_to_2(self.fit.errors['a2'])}\n" + f"#RESULTS=Tc={result_tmid_text}" + f",Rp/R*={result_rprs_text}" + f",inc={result_inc_text}" + f",{aavso_airmass_terms[0][0]}={aavso_airmass_terms[0][1]} +/- {aavso_airmass_terms[0][2]}" + f",{aavso_airmass_terms[1][0]}={aavso_airmass_terms[1][1]} +/- {aavso_airmass_terms[1][2]}\n" f"#RESULTS-XC={dumps(results_dict)}\n") # code yields + f.write(format_aavso_json_header("QC-XC", qc_metadata)) + f.write(format_aavso_json_header("FIT_QUALITY-XC", fit_quality_metadata)) + f.write(format_aavso_json_header("KTMF_DECISION-XC", ktmf_decision_metadata)) + f.write(format_aavso_json_header("PHOTOMETRY-XC", photometry_metadata)) + f.write(format_aavso_json_header("APERTURE-XC", aperture_metadata)) + f.write(format_aavso_json_header("FRAME_FILTERING-XC", frame_filtering_metadata)) + f.write(format_aavso_json_header("ASTROMETRY-XC", astrometry_metadata)) + f.write(format_aavso_json_header("BAD_PIXEL-XC", bad_pixel_metadata)) + f.write(format_aavso_json_header( + "OUT_OF_TRANSIT_BASELINE-XC", + out_of_transit_baseline_detrending_metadata(self.fit), + )) + f.write(format_aavso_json_header("DETREND_PARAMETERS-XC", { + 'DETREND_1': 'airmass', + 'DETREND_2': ( + 'out_of_transit_linear_baseline_correction_function' + if baseline_fixed_after_detrending(self.fit) + else 'airmass_correction_function' + ), + 'standard_header_preserved': True, + })) + f.write(format_aavso_json_header("MAGNITUDE_FIELDS-XC", { + 'apparent_magnitude': 'catalogue-calibrated target magnitude', + 'apparent_magnitude_error': 'flux and catalogue calibration uncertainty', + 'raw_differential_magnitude': ( + 'target minus selected comparison reference; ' + '-2.5 log10(target_flux/reference_flux)' + ), + 'raw_differential_magnitude_error': 'raw target/reference flux-only uncertainty', + 'corrected_differential_magnitude': ( + 'raw differential magnitude after the declared correction factor is removed' + ), + 'corrected_differential_magnitude_error': ( + 'flux-only uncertainty after correction; correction-model uncertainty excluded' + ), + 'differential_magnitude': ( + 'target minus selected comparison reference after the declared correction; ' + 'backward-compatible alias of corrected_differential_magnitude' + ), + 'differential_magnitude_error': ( + 'backward-compatible alias of corrected_differential_magnitude_error' + ), + 'differential_magnitude_correction_factor': ( + 'relative multiplicative flux correction; corrected_flux = raw_flux / factor' + ), + 'per_point_header': 'MAGNITUDE-XC', + 'band': magnitude_series['band'] or self.i_dict.get('filter'), + 'airmass_corrected': magnitude_series['airmass_corrected'], + 'correction_applied': magnitude_series['correction_applied'], + 'correction_type': magnitude_series['correction_type'], + 'raw_measurement_available': magnitude_series['raw_measurement_available'], + 'apparent_calibrated': magnitude_series['apparent_calibrated'], + 'comparison_reference': ( + getattr(self.fit, 'differential_magnitude_reference_label', None) + or getattr(self.fit, 'stellar_variability_reference_label', None) + or 'selected comparison reference' + ), + })) + for magnitude_index in range(0, len(self.fit.time)): + f.write(format_aavso_json_header("MAGNITUDE-XC", { + 'date_bjd_tdb': finite_float(self.fit.time[magnitude_index]), + 'raw_differential_magnitude': rounded_magnitude_value( + magnitude_series['raw_differential_magnitude'][magnitude_index], + digits=MAGNITUDE_DECIMAL_PLACES, + ), + 'raw_differential_magnitude_error': rounded_magnitude_error( + magnitude_series['raw_differential_magnitude_error'][magnitude_index], + digits=MAGNITUDE_DECIMAL_PLACES, + ), + 'corrected_differential_magnitude': rounded_magnitude_value( + magnitude_series['corrected_differential_magnitude'][magnitude_index], + digits=MAGNITUDE_DECIMAL_PLACES, + ), + 'corrected_differential_magnitude_error': rounded_magnitude_error( + magnitude_series['corrected_differential_magnitude_error'][magnitude_index], + digits=MAGNITUDE_DECIMAL_PLACES, + ), + 'differential_magnitude': rounded_magnitude_value( + magnitude_series['differential_magnitude'][magnitude_index], + digits=MAGNITUDE_DECIMAL_PLACES, + ), + 'differential_magnitude_correction_factor': finite_float( + magnitude_series['differential_magnitude_correction_factor'][magnitude_index] + ), + 'differential_magnitude_error': rounded_magnitude_error( + magnitude_series['differential_magnitude_error'][magnitude_index], + digits=MAGNITUDE_DECIMAL_PLACES, + ), + 'apparent_magnitude': rounded_magnitude_value( + magnitude_series['apparent_magnitude'][magnitude_index], + digits=MAGNITUDE_DECIMAL_PLACES, + ), + 'apparent_magnitude_error': rounded_magnitude_error( + magnitude_series['apparent_magnitude_error'][magnitude_index], + digits=MAGNITUDE_DECIMAL_PLACES, + ), + 'band': magnitude_series['band'] or self.i_dict.get('filter'), + }, preserve_nulls=True)) if epw_md5: f.write(f"#EPW_MD5-XC={dumps({'epw_checkout_md5': epw_md5})}\n") @@ -132,11 +3422,22 @@ def aavso(self, comp_star, airmasses, ld0, ld1, ld2, ld3, epw_md5): # f.write(f"{round(self.fit.time[aavsoC], 8)},{round(self.fit.data[aavsoC] / self.fit.parameters['a1'], 7)}," # f"{round(self.fit.dataerr[aavsoC] / self.fit.parameters['a1'], 7)},{round(airmasses[aavsoC], 7)}," # f"{round(self.fit.airmass_model[aavsoC] / self.fit.parameters['a1'], 7)}\n") - f.write(f"{round(self.fit.time[aavsoC], 8)},{round(self.fit.data[aavsoC], 7)}," - f"{round(self.fit.dataerr[aavsoC], 7)},{round(airmasses[aavsoC], 7)}," - f"{round(self.fit.airmass_model[aavsoC], 7)}\n") + f.write(f"{round(self.fit.time[aavsoC], 8)},{round(undetrended_flux[aavsoC], 7)}," + f"{round(undetrended_flux_error[aavsoC], 7)},{round(airmasses[aavsoC], 7)}," + f"{round(detrend_model[aavsoC], 7)}\n") + copy_aavso_supporting_artifacts( + self.dir, + self.p_dict['pName'], + self.i_dict['date'], + ) + def plate_status(self, plate_status: PlateStatus): - plate_status_file = self.dir / "temp" / f"PlateStatus_{self.p_dict['pName']}_{self.i_dict['date']}.csv" + plate_status_file = self.dir / "working_artifacts" / safe_output_filename( + "PlateStatus", + self.p_dict['pName'], + filename_date_token(self.i_dict['date']), + extension="csv", + ) plate_status.writePlateStatus(plate_status_file) class AIDOutputFiles: @@ -149,15 +3450,43 @@ def __init__(self, fit, p_dict, i_dict, auid, chart_id, vsp_params): self.dir = Path(self.i_dict['save']) self.vsp_params = vsp_params - def aavso(self): - params_file = self.dir / f"AID_AAVSO_{self.p_dict['sName']}_{self.i_dict['date']}.txt" + def _aavso_path(self): + return aavso_output_directory(self.dir) / safe_output_filename( + "AID_AAVSO", + self.p_dict['sName'], + filename_date_token(self.i_dict['date']), + extension="txt", + ) + + def _write_aavso(self, params_file, use_row_names=False, include_comparison_metadata=True): + first_vsp_param = self.vsp_params[0] if self.vsp_params else {} + comparison_metadata = aid_comparison_metadata(first_vsp_param) + ensemble_comparison_metadata = aid_ensemble_comparison_metadata(first_vsp_param) + fallback_differential_series = differential_magnitude_series_from_fit( + self.fit, + apply_airmass_correction=False, + ) + fallback_times = np.asarray( + [] if fallback_differential_series is None else fallback_differential_series['time'], + dtype=float, + ) + comparison_coordinate_headers = aid_comparison_coordinate_headers( + self.vsp_params, + indexed=use_row_names, + ) + default_variable_name = self.auid or self.p_dict.get('sName') or self.p_dict.get('pName') + with params_file.open('w', encoding="utf-8") as f: f.write("#TYPE=EXTENDED\n" # fixed f"#OBSCODE={self.i_dict['aavso_num']}\n" # UI f"#SOFTWARE=EXOTIC v{__version__}\n" # fixed "#DELIM=,\n" # fixed - "#DATE=JD\n" # fixed - f"#OBSTYPE={self.i_dict['camera']}\n") + "#DATE=BJD_TDB\n" # fixed + f"#OBSDATE={format_aavso_header_value(self.i_dict.get('date'))}\n" + f"#OBSTYPE={self.i_dict['camera']}\n" + f"#OBSLAT={format_aavso_header_value(self.i_dict.get('lat'))}\n" + f"#OBSLON={format_aavso_header_value(self.i_dict.get('long'))}\n" + f"#OBSELEV={format_aavso_header_value(self.i_dict.get('elev'))}\n") f.write( "# EXOTIC is developed by Exoplanet Watch (exoplanets.nasa.gov/exoplanet-watch/), a citizen science " "project managed by NASA's Jet Propulsion Laboratory on behalf of NASA's Universe of Learning. " @@ -165,90 +3494,267 @@ def aavso(self): "Space Telescope Science Institute.\n" "# Use of this data is governed by the AAVSO Data Usage Guidelines: " "aavso.org/data-usage-guidelines\n") + if include_comparison_metadata and comparison_metadata: + f.write(f"#COMPARISON-CATALOG-XC={dumps(comparison_metadata, sort_keys=True)}\n") + if comparison_coordinate_headers: + f.write(comparison_coordinate_headers) + if include_comparison_metadata and ensemble_comparison_metadata: + f.write(format_aavso_json_header( + "ENSEMBLE-COMPARISONS-XC", + ensemble_comparison_metadata, + )) + f.write(format_aavso_json_header("MAGNITUDE_FIELDS-XC", { + 'apparent_magnitude': 'MAG', + 'apparent_magnitude_error': 'MERR', + 'differential_magnitude': 'DIFFMAG', + 'differential_magnitude_error': 'DIFFERR', + 'differential_magnitude_definition': ( + 'target minus selected comparison reference; ' + '-2.5 log10(target_flux/reference_flux)' + ), + 'airmass_corrected': False, + })) - f.write("#NAME,DATE,MAG,MERR,FILT,TRANS,MTYPE,CNAME,CMAG,KNAME,KMAG,AMASS,GROUP,CHART,NOTES\n") + f.write( + "#NAME,DATE,MAG,MERR,FILT,TRANS,MTYPE,CNAME,CMAG,KNAME,KMAG," + "AMASS,GROUP,CHART,NOTES,DIFFMAG,DIFFERR\n" + ) for vsp_p in self.vsp_params: - f.write(f"{self.auid},{round(vsp_p['time'], 5)},{round(vsp_p['mag'], 5)},{round(vsp_p['mag_err'], 5)}," - f"{self.i_dict['filter']},NO,STD,{vsp_p['cname']},{round(vsp_p['cmag'], 5)},na,na," - f"{round(vsp_p['airmass'], 7)},na,{self.chart_id},na\n") + variable_name = default_variable_name + if use_row_names: + variable_name = vsp_p.get('_aid_name') or variable_name + mag = format_magnitude( + vsp_p.get('mag'), + default=None, + digits=MAGNITUDE_DECIMAL_PLACES, + ) + if mag is None: + continue + mag_err = format_magnitude_error( + vsp_p.get('mag_err'), + digits=MAGNITUDE_DECIMAL_PLACES, + ) + cmag = format_magnitude(vsp_p.get('cmag')) + chart_id = self.chart_id or vsp_p.get('chart_id') or 'na' + differential_mag, differential_error = differential_magnitude_from_vsp_param( + vsp_p + ) + if not np.isfinite(differential_mag) and fallback_times.size: + row_time = finite_float(vsp_p.get('time')) + time_matches = np.flatnonzero(np.isclose( + fallback_times, + row_time, + rtol=0.0, + atol=5.0e-5, + )) + if time_matches.size: + fallback_index = time_matches[0] + differential_mag = fallback_differential_series['magnitude'][fallback_index] + differential_error = ( + fallback_differential_series['magnitude_error'][fallback_index] + ) + differential_mag_text = format_magnitude( + differential_mag, + default=None, + digits=MAGNITUDE_DECIMAL_PLACES, + ) + differential_error_text = format_magnitude_error( + differential_error, + default=None, + digits=MAGNITUDE_DECIMAL_PLACES, + ) + differential_mag_text = differential_mag_text or 'na' + differential_error_text = differential_error_text or 'na' + f.write(f"{variable_name},{round(vsp_p['time'], 5)},{mag},{mag_err}," + f"{self.i_dict['filter']},NO,STD,{vsp_p['cname']},{cmag},na,na," + f"{round(vsp_p['airmass'], 7)},na,{chart_id},na," + f"{differential_mag_text},{differential_error_text}\n") + return params_file + + def aavso(self): + params_file = self._write_aavso(self._aavso_path()) + copy_aavso_supporting_artifacts( + self.dir, + self.p_dict.get('pName') or self.p_dict.get('sName'), + self.i_dict['date'], + ) + return params_file + + def combined_aavso(self): + """Write one AID file containing rows for multiple named variables.""" + params_file = self._write_aavso( + self._aavso_path(), + use_row_names=True, + include_comparison_metadata=False, + ) + copy_aavso_supporting_artifacts( + self.dir, + self.p_dict.get('pName') or self.p_dict.get('sName'), + self.i_dict['date'], + ) + return params_file def aavso_dicts(planet_dict, fit, info_dict, durs, ld0, ld1, ld2, ld3): + aavso_airmass_terms = aavso_airmass_results(fit) + rprs_report_error = fit_rprs_report_error(fit) + if not np.isfinite(rprs_report_error) or rprs_report_error < 0: + rprs_report_error = finite_float(planet_dict.get('rprsUnc')) priors = { - 'Period': { - 'value': str(round_to_2(planet_dict['pPer'], planet_dict['pPerUnc'])), - 'uncertainty': str(round_to_2(planet_dict['pPerUnc'])) if planet_dict['pPerUnc'] else planet_dict['pPerUnc'], - 'units': "days" - }, - 'Rp/R*': { - 'value': str(round_to_2(planet_dict['rprs'], planet_dict['rprsUnc'])), - 'uncertainty': str(round_to_2(planet_dict['rprsUnc'])) if planet_dict['rprsUnc'] else planet_dict['rprsUnc'], - }, - 'a/R*': { - 'value': str(round_to_2(planet_dict['aRs'], planet_dict['aRsUnc'])), - 'uncertainty': str(round_to_2(planet_dict['aRsUnc'])) if planet_dict['aRsUnc'] else planet_dict['aRsUnc'], - }, - 'inc': { - 'value': str(round_to_2(planet_dict['inc'], planet_dict['incUnc'])), - 'uncertainty': str(round_to_2(planet_dict['incUnc'])) if planet_dict['incUnc'] else planet_dict['incUnc'], - 'units': "degrees" - }, + 'Period': aavso_result_entry(planet_dict['pPer'], planet_dict['pPerUnc'], units="days"), + 'Rp/R*': aavso_result_entry(planet_dict['rprs'], planet_dict['rprsUnc']), + 'a/R*': aavso_result_entry(planet_dict['aRs'], planet_dict['aRsUnc']), + 'inc': aavso_result_entry(planet_dict['inc'], planet_dict['incUnc'], units="degrees"), 'ecc': { 'value': str(round_to_2(planet_dict['ecc'])), 'uncertainty': None, }, - 'u0': { - 'value': str(round_to_2(ld0[0], ld0[1])), - 'uncertainty': str(round_to_2(ld0[1])) - }, - 'u1': { - 'value': str(round_to_2(ld1[0], ld1[1])), - 'uncertainty': str(round_to_2(ld1[1])) - }, - 'u2': { - 'value': str(round_to_2(ld2[0], ld2[1])), - 'uncertainty': str(round_to_2(ld2[1])) - }, - 'u3': { - 'value': str(round_to_2(ld3[0], ld3[1])), - 'uncertainty': str(round_to_2(ld3[1])) - } + 'u0': aavso_result_entry(*ld0), + 'u1': aavso_result_entry(*ld1), + 'u2': aavso_result_entry(*ld2), + 'u3': aavso_result_entry(*ld3), } filter_type = { 'name': info_dict['filter'], 'desc': info_dict['filter_desc'], - 'fwhm': [{'value': str(info_dict['wl_min']) if info_dict['wl_min'] else info_dict['wl_min'], 'units': "nm"}, - {'value': str(info_dict['wl_max']) if info_dict['wl_max'] else info_dict['wl_max'], 'units': "nm"}], + 'filter_width': { + 'left_side_wavelength': { + 'value': str(info_dict['wl_min']) if info_dict['wl_min'] else info_dict['wl_min'], + 'units': "nm" + }, + 'right_side_wavelength': { + 'value': str(info_dict['wl_max']) if info_dict['wl_max'] else info_dict['wl_max'], + 'units': "nm" + } + }, } results = { - 'Tc': { - 'value': str(round_to_2(fit.parameters['tmid'], fit.errors['tmid'])), - 'uncertainty': str(round_to_2(fit.errors['tmid'])), - 'units': "BJD_TDB" - }, - 'Rp/R*': { - 'value': str(round_to_2(fit.parameters['rprs'], fit.errors['rprs'])), - 'uncertainty': str(round_to_2(fit.errors['rprs'])) - }, - 'inc': { - 'value': str(round_to_2(fit.parameters['inc'], fit.errors['inc'])), - 'uncertainty': str(round_to_2(fit.errors['inc'])), - }, - 'Am1': { - 'value': str(round_to_2(fit.parameters['a1'], fit.errors['a1'])), - 'uncertainty': str(round_to_2(fit.errors['a1'])) - }, + 'Tc': aavso_result_entry( + fit.parameters['tmid'], fit.errors['tmid'], units="BJD_TDB" + ), + 'Rp/R*': aavso_result_entry(fit.parameters['rprs'], rprs_report_error), + 'inc': aavso_result_entry(fit.parameters['inc'], fit.errors['inc']), 'Am2': { - 'value': str(round_to_2(fit.parameters['a2'], fit.errors['a2'])), - 'uncertainty': str(round_to_2(fit.errors['a2'])) + 'value': aavso_airmass_terms[1][1], + 'uncertainty': aavso_airmass_terms[1][2] }, - 'Duration': { - 'value': str(round_to_2(mean(durs))), - 'uncertainty': str(round_to_2(std(durs))), - 'units': "days" - } + 'Duration': aavso_result_entry(mean(durs), std(durs), units="days"), } + results[aavso_airmass_terms[0][0]] = { + 'value': aavso_airmass_terms[0][1], + 'uncertainty': aavso_airmass_terms[0][2] + } + optional_results = { + 'a/R*': aavso_result_entry( + fit.parameters.get('ars'), + fit.errors.get('ars'), + ), + } + impact_parameter, impact_error = fit_impact_parameter_value_error(fit) + optional_results['Impact Parameter (b)'] = aavso_result_entry(impact_parameter, impact_error) + + limb_darkening = (ld0, ld1, ld2, ld3) + prior_parameters = planet_dict_transit_parameters( + planet_dict, + limb_darkening=limb_darkening, + fallback=getattr(fit, 'prior', None), + ) + prior_errors = planet_dict_transit_errors(planet_dict, limb_darkening=limb_darkening) + depth_summary = fit_transit_depth_summary( + fit, + prior_parameters=prior_parameters, + prior_errors=prior_errors, + ) + for label, value_key, error_key in ( + (AREA_DEPTH_LABEL, 'area_depth', 'area_depth_error'), + (OBSERVABLE_DEPTH_LABEL, 'observable_depth', 'observable_depth_error'), + (PRIOR_OBSERVABLE_DEPTH_LABEL, 'prior_observable_depth', 'prior_observable_depth_error'), + (OBSERVABLE_DEPTH_DELTA_LABEL, 'observable_depth_prior_delta', 'observable_depth_prior_delta_error'), + ): + optional_results[label] = aavso_result_entry( + depth_summary.get(value_key), + depth_summary.get(error_key), + units="percent", + ) + + scatter = residual_scatter_fraction(fit) + optional_results['Residual scatter around full model fit'] = aavso_result_entry( + 100.0 * scatter if np.isfinite(scatter) else np.nan, + units="percent", + ) + if 'a0' in fit.parameters: + optional_results['a0'] = aavso_result_entry(fit.parameters.get('a0'), fit.errors.get('a0')) + elif 'a1' in fit.parameters: + optional_results['a1'] = aavso_result_entry(fit.parameters.get('a1'), fit.errors.get('a1')) + + results.update({ + key: value for key, value in optional_results.items() + if value is not None + }) + return priors, filter_type, results + + +def format_aavso_header_value(value): + if value is None: + return "" + if isinstance(value, str): + stripped = value.strip() + return "" if stripped.lower() in ('', 'n/a', 'na', 'null', 'none') else stripped + return str(value) + + +def save_comp_star_calibration_summary(save_dir, target_name, date, method_label, field_score, + comp_summaries, best_comp_index): + temp_dir = Path(save_dir) / "working_artifacts" + temp_dir.mkdir(parents=True, exist_ok=True) + summary_file = temp_dir / safe_output_filename( + "CompStarCalibrationSummary", + target_name, + filename_date_token(date), + extension="csv", + ) + + with summary_file.open('w') as handle: + handle.write(f"# Comparison-star calibration summary for {target_name}\n") + handle.write(f"# Method,{method_label}\n") + if field_score is not None and field_score == field_score: + handle.write(f"# Field suitability score,{field_score}\n") + else: + handle.write("# Field suitability score,\n") + handle.write(f"# Selected comparison star,{'' if best_comp_index is None else best_comp_index + 1}\n") + handle.write("comp_star,x_pixel,y_pixel,selected,suitability_score,intercomparison_score,pairwise_median_score," + "pairwise_max_score,self_score,valid_pair_count,coverage_count,coverage_peer_median," + "coverage_min_required,coverage_rejected,suitability_outlier_rejected," + "psf_quality_rejected_count,overexposure_rejected_count,intercomparison_frame_rejected_count," + "intercomparison_frame_required_valid_pairs\n") + + for summary in comp_summaries: + position = summary.get('position') or [None, None] + values = [ + summary.get('label', ''), + position[0], + position[1], + str(bool(summary.get('selected'))).lower(), + summary.get('aggregate_score'), + summary.get('ensemble_score'), + summary.get('pairwise_median_score'), + summary.get('pairwise_max_score'), + summary.get('self_score'), + summary.get('valid_pair_count'), + summary.get('coverage_count'), + summary.get('coverage_reference_count'), + summary.get('coverage_min_required_count'), + summary.get('coverage_rejected'), + summary.get('suitability_outlier_rejected'), + summary.get('psf_quality_rejected_count', 0), + summary.get('overexposure_rejected_count', 0), + summary.get('ensemble_frame_rejected_count', 0), + summary.get('ensemble_frame_required_valid_pairs', 0), + ] + handle.write(",".join("" if value is None else str(value) for value in values) + "\n") + + return summary_file diff --git a/exotic/plate_status.py b/exotic/plate_status.py index ff100a67..eb257752 100644 --- a/exotic/plate_status.py +++ b/exotic/plate_status.py @@ -1,13 +1,26 @@ class PlateStatus: + WARNING_PROGRESS_INTERVAL = 100 + WARNING_CONDITION_TEXT = { + "outofframe": "outside the image", + "lowflux": "low flux", + "overexposed": "overexposed", + "skybg": "a sky-background failure", + } + def __init__(self, logfunc): self.statusByFilename = dict() self.filenameList = [] self.filename = "N/A" self.errorcodes = set() self.logfunc = logfunc + self.comparisonStarLabels = {} + self.aggregatedWarnings = {} + self.lastAggregatedWarningSummary = {} + self.aggregationNoticeLogged = False self.errorcodes.add("outofframe_target") self.errorcodes.add("lowflux_target") + self.errorcodes.add("overexposed_target") self.errorcodes.add("skybg_target") self.errorcodes.add("fits_error") self.errorcodes.add("alignment_error") @@ -24,7 +37,34 @@ def initializeComparisonStarCount(self, compCount: int): for i in range(compCount): self.errorcodes.add(f"outofframe_comp{i+1}") self.errorcodes.add(f"lowflux_comp{i+1}") - self.errorcodes.add(f"skybg_comp{i+1}") + self.errorcodes.add(f"overexposed_comp{i+1}") + self.errorcodes.add(f"skybg_comp{i+1}") + + def setComparisonStarLabels(self, labels=None): + normalized_labels = {} + for star_index, label in (labels or {}).items(): + try: + normalized_index = int(star_index) + except (TypeError, ValueError): + continue + if normalized_index <= 0 or not isinstance(label, str) or not label.strip(): + continue + normalized_labels[normalized_index] = label.strip() + self.comparisonStarLabels = normalized_labels + return self + + def _starLabel(self, starIndex: int): + if starIndex == 0: + return "Target star" + return self.comparisonStarLabels.get(starIndex, f"Comparison star #{starIndex}") + + def _displayFilename(self): + return str(self.filename).replace('\\', '/').rsplit('/', 1)[-1] + + def _conditionCountText(self, condition: str, count: int): + condition_text = self.WARNING_CONDITION_TEXT.get(condition, condition) + return f"{condition_text} in {count} frame(s)" + # Sets current filename (for any reported errors) - sets starIndex=0 (target) def setCurrentFilename(self, filename: str): filename = str(filename) @@ -33,7 +73,8 @@ def setCurrentFilename(self, filename: str): self.filename = filename return self # Log an error - def _logError(self, errorcode: str, message: str) -> None: + def _logError(self, errorcode: str, message: str, starIndex: int = None, + condition: str = None, starLabel: str = None) -> None: if self.filename not in self.statusByFilename: self.statusByFilename[self.filename] = {} rec = self.statusByFilename[self.filename] @@ -42,32 +83,133 @@ def _logError(self, errorcode: str, message: str) -> None: # Mark error on this file rec[errorcode] = True self.errorcodes.add(errorcode) - # And log new warning - self.logfunc(message, warn=True) + if starIndex is None or condition is None: + self.logfunc(message, warn=True) + return + + label = ( + starLabel.strip() + if isinstance(starLabel, str) and starLabel.strip() + else self._starLabel(starIndex) + ) + aggregate = self.aggregatedWarnings.setdefault(errorcode, { + 'star_index': starIndex, + 'label': label, + 'condition': condition, + 'count': 0, + }) + aggregate['label'] = label + aggregate['count'] += 1 + warning_count = aggregate['count'] + + if warning_count == 1: + self.logfunc(message, warn=True) + if not self.aggregationNoticeLogged: + self.logfunc( + "Plate-status detail: repeated frame-level star warnings are aggregated after " + "their first occurrence; running counts are reported every " + f"{self.WARNING_PROGRESS_INTERVAL} frames and exact per-frame flags are preserved " + "in the PlateStatus CSV." + ) + self.aggregationNoticeLogged = True + elif warning_count % self.WARNING_PROGRESS_INTERVAL == 0: + self.logfunc( + "Plate-status warning update: " + f"{label}: {self._conditionCountText(condition, warning_count)} so far.", + warn=True, + ) + + def logAggregatedWarningSummary(self): + current_counts = { + errorcode: aggregate['count'] + for errorcode, aggregate in self.aggregatedWarnings.items() + } + if current_counts == self.lastAggregatedWarningSummary: + return + + repeated = [ + aggregate for aggregate in self.aggregatedWarnings.values() + if aggregate['count'] > 1 + ] + self.lastAggregatedWarningSummary = current_counts + if not repeated: + return + + grouped = {} + for aggregate in repeated: + group = grouped.setdefault(aggregate['star_index'], { + 'label': aggregate['label'], + 'conditions': [], + 'total': 0, + }) + group['label'] = aggregate['label'] + group['conditions'].append( + self._conditionCountText(aggregate['condition'], aggregate['count']) + ) + group['total'] += aggregate['count'] + + self.logfunc( + "Plate-status warning summary: repeated frame-level diagnostics were aggregated; " + "exact per-frame flags are preserved in the PlateStatus CSV." + ) + for group in sorted( + grouped.values(), + key=lambda item: (-item['total'], item['label']), + ): + self.logfunc(f">-- {group['label']}: {'; '.join(group['conditions'])}.") + # Report out of frame warning for start ;index' (0=target, 1+=comp #N) def outOfFrameWarning(self, starIndex): - if starIndex == 0: # Target star - self._logError("outofframe_target", - f"Target star beyond edge of file {self.filename}") - else: - self._logError(f"outofframe_comp{starIndex}", - f"Comparison star #{starIndex} star beyond edge of file {self.filename}") + label = self._starLabel(starIndex) + errorcode = "outofframe_target" if starIndex == 0 else f"outofframe_comp{starIndex}" + self._logError( + errorcode, + f"{label} is beyond the edge of file {self._displayFilename()}", + starIndex=starIndex, + condition="outofframe", + starLabel=label, + ) # Report low flux amplitude warning for start ;index' (0=target, 1+=comp #N) def lowFluxAmplitudeWarning(self, starIndex: int, xc: float, yc: float): - if starIndex == 0: # Target star - self._logError("lowflux_target", - f"Measured flux for Target star is low in file {self.filename} - are you sure there is a star at [{xc:.1f}, {yc:.1f}]?") - else: - self._logError(f"lowflux_comp{starIndex}", - f"Measured flux for Comparison star #{starIndex} is low in file {self.filename} - are you sure there is a star at [{xc:.1f}, {yc:.1f}]?") + label = self._starLabel(starIndex) + errorcode = "lowflux_target" if starIndex == 0 else f"lowflux_comp{starIndex}" + self._logError( + errorcode, + f"Measured flux for {label} is low in file {self._displayFilename()} - " + f"are you sure there is a star at [{xc:.1f}, {yc:.1f}]?", + starIndex=starIndex, + condition="lowflux", + starLabel=label, + ) + # Report overexposure warning for star index (0=target, 1+=comp #N) + def overexposedWarning(self, starIndex: int, xc: float, yc: float, threshold: float, + starLabel: str = None): + label = ( + starLabel.strip() + if isinstance(starLabel, str) and starLabel.strip() + else self._starLabel(starIndex) + ) + errorcode = "overexposed_target" if starIndex == 0 else f"overexposed_comp{starIndex}" + self._logError( + errorcode, + f"{label} is overexposed in file {self._displayFilename()}; aperture pixels near " + f"[{xc:.1f}, {yc:.1f}] exceeded {threshold:.1f}.", + starIndex=starIndex, + condition="overexposed", + starLabel=label, + ) # Report sky background warning for start ;index' (0=target, 1+=comp #N) def skyBackgroundWarning(self, starIndex: int, xc: float, yc: float): - if starIndex == 0: # Target star - self._logError("skybg_target", - f"Sky background error for Target star for file {self.filename} - are you sure there is a star at [{xc:.1f}, {yc:.1f}]?") - else: - self._logError(f"skybg_comp{starIndex}", - f"Sky background error for Comparison star #{starIndex} for file {self.filename} - are you sure there is a star at [{xc:.1f}, {yc:.1f}]?") + label = self._starLabel(starIndex) + errorcode = "skybg_target" if starIndex == 0 else f"skybg_comp{starIndex}" + self._logError( + errorcode, + f"Sky background error for {label} in file {self._displayFilename()} - " + f"are you sure there is a star at [{xc:.1f}, {yc:.1f}]?", + starIndex=starIndex, + condition="skybg", + starLabel=label, + ) # Reort file format error def fitsFormatError(self, e: OSError): self._logError("fits_error", @@ -84,6 +226,7 @@ def alignmentError(self): f"File {self.filename} failed to align with first file") # Write plate status to CSV file def writePlateStatus(self, file: str): + self.logAggregatedWarningSummary() with open(file, 'w') as f: cols = list(self.errorcodes) cols.sort() diff --git a/exotic/plots.py b/exotic/plots.py index 387b8311..eaca0f6a 100644 --- a/exotic/plots.py +++ b/exotic/plots.py @@ -1,14 +1,61 @@ from astropy.visualization import astropy_mpl_style, ZScaleInterval, ImageNormalize from astropy.visualization.stretch import LinearStretch, SquaredStretch, SqrtStretch, LogStretch +import inspect import matplotlib.patheffects as path_effects import matplotlib.pyplot as plt from matplotlib.lines import Line2D import numpy as np from pathlib import Path +try: + from utils import ( + filename_date_token, + format_value_with_uncertainty, + is_usable_apparent_magnitude, + magnitude_text, + normalized_magnitude_error, + safe_output_filename, + ) +except ImportError: + from .utils import ( + filename_date_token, + format_value_with_uncertainty, + is_usable_apparent_magnitude, + magnitude_text, + normalized_magnitude_error, + safe_output_filename, + ) + +try: + from output_files import ( + differential_magnitude_series_from_fit, + empirical_red_noise_error_scale, + fit_empirical_transit_uncertainty, + fit_impact_parameter_value_error, + fit_parameter_model_data_uncertainty, + ) +except ImportError: + from .output_files import ( + differential_magnitude_series_from_fit, + empirical_red_noise_error_scale, + fit_empirical_transit_uncertainty, + fit_impact_parameter_value_error, + fit_parameter_model_data_uncertainty, + ) + plt.style.use(astropy_mpl_style) +def _dated_plot_filename(prefix, *parts, date, extension): + return safe_output_filename(prefix, *parts, filename_date_token(date), extension=extension) + + +def _working_artifacts_dir(save): + output_dir = Path(save) / "working_artifacts" + output_dir.mkdir(parents=True, exist_ok=True) + return output_dir + + # Plots of the centroid positions as a function of time def plot_centroids(x_targ, y_targ, x_ref, y_ref, times, target_name, save, date): fig, axs = plt.subplots(3, 2, figsize=(12, 10)) @@ -46,16 +93,47 @@ def plot_centroids(x_targ, y_targ, x_ref, y_ref, times, target_name, save, date) axs[2, 1].plot(times[e] - np.nanmin(times), abs(y_targ[e] - y_ref[e]), 'k.') plt.tight_layout() - plt.savefig(Path(save) / "temp" / f"CentroidPositions&Distances_{target_name}_{date}.pdf") + plt.savefig(_working_artifacts_dir(save) / _dated_plot_filename( + "CentroidPositions&Distances", + target_name, + date=date, + extension="pdf", + )) plt.close() -def plot_fov(aper, annulus, sigma, x_targ, y_targ, x_ref, y_ref, image, image_scale, targ_name, save, date, opt_method, min_aper_fov, min_annulus_fov): +def plot_fov(aper, annulus, sigma, x_targ, y_targ, x_ref, y_ref, image, image_scale, targ_name, save, date, + opt_method, min_aper_fov, min_annulus_fov, sky_inner_radius=None, sky_outer_radius=None, + comparison_positions=None, comparison_labels=None): - ref_circle, ref_circle_sky = None, None - picframe = 10. * (aper + 15. * sigma) + if comparison_positions is None: + comparison_positions = [[x_ref, y_ref]] + valid_comparison_positions = [] + for position in comparison_positions: + try: + position_values = np.asarray(position, dtype=float).reshape(-1) + except (TypeError, ValueError): + continue + if position_values.size < 2 or not np.all(np.isfinite(position_values[:2])): + continue + valid_comparison_positions.append(( + float(position_values[0]), + float(position_values[1]), + )) + if aper < 0: + valid_comparison_positions = [] - pltx = [max([0, min([x_targ, x_ref]) - picframe]), min([np.shape(image)[1], max([x_targ, x_ref]) + picframe])] - plty = [max([0, min([y_targ, y_ref]) - picframe]), min([np.shape(image)[0], max([y_targ, y_ref]) + picframe])] + labels = list(comparison_labels or []) + if len(labels) != len(valid_comparison_positions): + if len(valid_comparison_positions) == 1: + labels = ['Comp Star'] + else: + labels = [f'Comp {index + 1}' for index in range(len(valid_comparison_positions))] + + picframe = 10. * (aper + 15. * sigma) + plotted_x = [x_targ, *(position[0] for position in valid_comparison_positions)] + plotted_y = [y_targ, *(position[1] for position in valid_comparison_positions)] + pltx = [max(0, min(plotted_x) - picframe), min(np.shape(image)[1], max(plotted_x) + picframe)] + plty = [max(0, min(plotted_y) - picframe), min(np.shape(image)[0], max(plotted_y) + picframe)] for stretch in [LinearStretch(), SquaredStretch(), SqrtStretch(), LogStretch()]: fig, ax = plt.subplots() @@ -68,13 +146,21 @@ def plot_fov(aper, annulus, sigma, x_targ, y_targ, x_ref, y_ref, image, image_sc # Create the target circles # We are using abs(aper) to account for a negative aperture in case EXOTIC is not using a comparison star - target_circle = plt.Circle((x_targ, y_targ), abs(aper), color=outer_circle_color, fill=False, ls='-') - target_circle_sky = plt.Circle((x_targ, y_targ), abs(aper) + annulus, color=outer_circle_color, fill=False, ls='-') + if sky_inner_radius is None or sky_outer_radius is None: + local_sky_inner_radius = abs(aper) + 2.0 + if np.isfinite(sigma) and sigma > 0: + local_sky_inner_radius = max(local_sky_inner_radius, 3.0 * 2.355 * float(sigma)) + local_sky_outer_radius = max( + local_sky_inner_radius + annulus, + np.sqrt(local_sky_inner_radius ** 2 + 250.0 / np.pi), + ) + else: + local_sky_inner_radius = float(sky_inner_radius) + local_sky_outer_radius = float(sky_outer_radius) - # IF EXOTIC is using a comparison star, create its circles - if aper >= 0: - ref_circle = plt.Circle((x_ref, y_ref), aper, color=outer_circle_color, fill=False, ls='-') - ref_circle_sky = plt.Circle((x_ref, y_ref), aper + annulus, color=outer_circle_color, fill=False, ls='-') + target_circle = plt.Circle((x_targ, y_targ), abs(aper), color=outer_circle_color, fill=False, ls='-') + target_circle_sky_inner = plt.Circle((x_targ, y_targ), local_sky_inner_radius, color=outer_circle_color, fill=False, ls='--') + target_circle_sky_outer = plt.Circle((x_targ, y_targ), local_sky_outer_radius, color=outer_circle_color, fill=False, ls='-') interval = ZScaleInterval() vmin, vmax = interval.get_limits(image) @@ -85,18 +171,46 @@ def plot_fov(aper, annulus, sigma, x_targ, y_targ, x_ref, y_ref, image, image_sc fig.colorbar(im) ax.add_artist(target_circle) - ax.add_artist(target_circle_sky) - ax.text(x_targ + abs(aper) + annulus + 5, y_targ, targ_name, color='w', fontsize=10, + ax.add_artist(target_circle_sky_inner) + ax.add_artist(target_circle_sky_outer) + ax.text(x_targ + local_sky_outer_radius + 5, y_targ, targ_name, color='w', fontsize=10, path_effects=[path_effects.withStroke(linewidth=2, foreground='black')]) - if aper >= 0: #EXOTIC is using a comparison star + for (comparison_x, comparison_y), comparison_label in zip( + valid_comparison_positions, labels): + ref_circle = plt.Circle( + (comparison_x, comparison_y), aper, + color=outer_circle_color, fill=False, ls='-' + ) + ref_circle_sky_inner = plt.Circle( + (comparison_x, comparison_y), local_sky_inner_radius, + color=outer_circle_color, fill=False, ls='--' + ) + ref_circle_sky = plt.Circle( + (comparison_x, comparison_y), local_sky_outer_radius, + color=outer_circle_color, fill=False, ls='-' + ) ax.add_artist(ref_circle) + ax.add_artist(ref_circle_sky_inner) ax.add_artist(ref_circle_sky) - ax.text(x_ref + aper + annulus + 5, y_ref, 'Comp Star', color='w', fontsize=10, - path_effects=[path_effects.withStroke(linewidth=2, foreground='black')]) + ax.text( + comparison_x + local_sky_outer_radius + 5, + comparison_y, + comparison_label, + color='w', + fontsize=10, + path_effects=[path_effects.withStroke(linewidth=2, foreground='black')], + ) handles = [] - label_aper = f"{opt_method} Photometry\n(Min Aper: {abs(min_aper_fov):.2f} px)\n(Min Annulus: {min_annulus_fov:.2f} px)" + if opt_method == "PSF": + label_aper = "PSF Photometry" + else: + label_aper = ( + f"{opt_method} Photometry\n" + f"(Min Aper: {abs(min_aper_fov):.2f} px)\n" + f"(Min Annulus: {min_annulus_fov:.2f} px)" + ) if opt_method == "Aperture": aperture_line = Line2D([], [], color=outer_circle_color, linestyle='-', label=label_aper) @@ -120,12 +234,23 @@ def plot_fov(aper, annulus, sigma, x_targ, y_targ, x_ref, y_ref, image, image_sc apos = '\'' Path(save).mkdir(parents=True, exist_ok=True) - Path(save, "temp").mkdir(parents=True, exist_ok=True) + _working_artifacts_dir(save) - plt.savefig(Path(save) / "temp" / f"FOV_{targ_name}_{date}_" - f"{str(stretch.__class__).split('.')[-1].split(apos)[0]}.pdf", bbox_inches='tight') - plt.savefig(Path(save) / "temp" / f"FOV_{targ_name}_{date}_" - f"{str(stretch.__class__).split('.')[-1].split(apos)[0]}.png", bbox_inches='tight') + stretch_name = str(stretch.__class__).split('.')[-1].split(apos)[0] + plt.savefig(_working_artifacts_dir(save) / _dated_plot_filename( + "FOV", + targ_name, + stretch_name, + date=date, + extension="pdf", + ), bbox_inches='tight') + plt.savefig(_working_artifacts_dir(save) / _dated_plot_filename( + "FOV", + targ_name, + stretch_name, + date=date, + extension="png", + ), bbox_inches='tight') plt.close() @@ -135,7 +260,7 @@ def plot_flux(times, targ, targ_unc, ref, ref_unc, norm_flux, norm_unc, airmass, plt.xlabel("Time [BJD_TDB]") plt.ylabel("Flux [ADU]") plt.errorbar(times, targ, yerr=targ_unc, linestyle='None', fmt='-o') - plt.savefig(Path(save) / "temp" / f"TargetRawFlux_{targ_name}_{date}.pdf") + plt.savefig(_working_artifacts_dir(save) / _dated_plot_filename("TargetRawFlux", targ_name, date=date, extension="pdf")) plt.close() plt.figure() @@ -143,7 +268,7 @@ def plot_flux(times, targ, targ_unc, ref, ref_unc, norm_flux, norm_unc, airmass, plt.xlabel("Time [BJD_TDB]") plt.ylabel("Flux [ADU]") plt.errorbar(times, ref, yerr=ref_unc, linestyle='None', fmt='-o') - plt.savefig(Path(save) / "temp" / f"CompRawFlux_{targ_name}_{date}.pdf") + plt.savefig(_working_artifacts_dir(save) / _dated_plot_filename("CompRawFlux", targ_name, date=date, extension="pdf")) plt.close() # Plots final reduced light curve (after the 3 sigma clip) @@ -152,11 +277,11 @@ def plot_flux(times, targ, targ_unc, ref, ref_unc, norm_flux, norm_unc, airmass, plt.xlabel("Time [BJD_TDB]") plt.ylabel("Normalized Flux") plt.errorbar(times, norm_flux, yerr=norm_unc, linestyle='None', fmt='-bo') - plt.savefig(Path(save) / "temp" / f"NormalizedFluxTime_{targ_name}_{date}.pdf") + plt.savefig(_working_artifacts_dir(save) / _dated_plot_filename("NormalizedFluxTime", targ_name, date=date, extension="pdf")) plt.close() # Save normalized flux to text file prior to NS - params_file = Path(save) / "temp" / f"NormalizedFlux_{targ_name}_{date}.txt" + params_file = _working_artifacts_dir(save) / _dated_plot_filename("NormalizedFlux", targ_name, date=date, extension="txt") with params_file.open('w') as f: f.write("BJD,Norm Flux,Norm Err,AM\n") @@ -164,28 +289,576 @@ def plot_flux(times, targ, targ_unc, ref, ref_unc, norm_flux, norm_unc, airmass, f.write(f"{round(ti, 8)},{round(fi, 7)},{round(erri, 6)},{round(ami, 2)}\n") +def plot_comp_star_pairwise_matrix(pairwise_matrix, best_comp_index, targ_name, save, date, method_label): + matrix = np.asarray(pairwise_matrix, dtype=float) + if matrix.size == 0: + return + + temp_dir = _working_artifacts_dir(save) + + fig, ax = plt.subplots(figsize=(max(6, matrix.shape[0] * 1.3), max(5, matrix.shape[0] * 1.1))) + plot_matrix = np.ma.masked_invalid(matrix * 100.0) + im = ax.imshow(plot_matrix, origin='upper', cmap='viridis') + fig.colorbar(im, ax=ax, label="Residual Scatter [%]") + + labels = [f"Comp {index + 1}" for index in range(matrix.shape[0])] + ax.set_xticks(np.arange(matrix.shape[0])) + ax.set_yticks(np.arange(matrix.shape[0])) + ax.set_xticklabels(labels, rotation=45, ha='right') + ax.set_yticklabels(labels) + ax.set_title(f"{targ_name} Comparison-Star Pairwise Scatter\n{method_label}") + + for row in range(matrix.shape[0]): + for col in range(matrix.shape[1]): + value = matrix[row, col] + if np.isfinite(value): + ax.text(col, row, f"{value * 100.0:.3f}", ha='center', va='center', color='white', fontsize=8) + + if best_comp_index is not None and 0 <= best_comp_index < matrix.shape[0]: + ax.add_patch(plt.Rectangle((best_comp_index - 0.5, best_comp_index - 0.5), 1, 1, + fill=False, edgecolor='tomato', linewidth=2.5)) + + ax.set_xlabel("Reference Comparison Star") + ax.set_ylabel("Candidate Comparison Star") + fig.tight_layout() + fig.savefig(temp_dir / _dated_plot_filename("CompStarPairwiseScatter", targ_name, date=date, extension="png"), bbox_inches="tight") + fig.savefig(temp_dir / _dated_plot_filename("CompStarPairwiseScatter", targ_name, date=date, extension="pdf"), bbox_inches="tight") + plt.close(fig) + + +def plot_comp_star_calibration_series(times, comp_summaries, targ_name, save, date, method_label): + if not comp_summaries: + return + + times = np.asarray(times, dtype=float) + temp_dir = _working_artifacts_dir(save) + colors = plt.cm.tab10(np.linspace(0.0, 1.0, 10)) + + fig_height = max(3.2, 2.4 * len(comp_summaries)) + fig, axes = plt.subplots(len(comp_summaries), 1, figsize=(12, fig_height), sharex=True) + if len(comp_summaries) == 1: + axes = [axes] + + for axis, summary in zip(axes, comp_summaries): + _draw_comp_star_calibration_axis(axis, times, summary, colors) + + axes[-1].set_xlabel("Time [BJD_TDB]") + fig.suptitle(f"{targ_name} Comparison-Star Calibration Curves\n{method_label}", y=1.01) + fig.tight_layout() + fig.savefig(temp_dir / _dated_plot_filename("CompStarCalibrationCurves", targ_name, date=date, extension="png"), bbox_inches="tight") + fig.savefig(temp_dir / _dated_plot_filename("CompStarCalibrationCurves", targ_name, date=date, extension="pdf"), bbox_inches="tight") + plt.close(fig) + + +def plot_individual_comp_star_calibration_series(times, comp_summaries, targ_name, save, date, method_label): + if not comp_summaries: + return + + times = np.asarray(times, dtype=float) + temp_dir = _working_artifacts_dir(save) + colors = plt.cm.tab10(np.linspace(0.0, 1.0, 10)) + + for summary in comp_summaries: + fig, axis = plt.subplots(figsize=(12, 4)) + _draw_comp_star_calibration_axis(axis, times, summary, colors) + axis.set_xlabel("Time [BJD_TDB]") + fig.suptitle(f"{targ_name} {summary['label']} Calibration Curves\n{method_label}") + fig.tight_layout() + label_slug = summary['label'].replace(" ", "") + fig.savefig(temp_dir / _dated_plot_filename( + "CompStarCalibrationCurve", + label_slug, + targ_name, + date=date, + extension="png", + ), bbox_inches="tight") + fig.savefig(temp_dir / _dated_plot_filename( + "CompStarCalibrationCurve", + label_slug, + targ_name, + date=date, + extension="pdf", + ), bbox_inches="tight") + plt.close(fig) + + +def plot_comp_star_candidate_lightcurve_fits(candidate_fit_summaries, targ_name, save, date, method_label): + if not candidate_fit_summaries: + return + + temp_dir = _working_artifacts_dir(save) + + for summary in candidate_fit_summaries: + fit = summary.get('fit') + if fit is None: + continue + + fig, (ax_lc, ax_res) = _plot_bestfit_for_lightcurve_png( + fit, + phase=False, + show_flux_baseline_label=False, + ) + selected_text = " selected" if summary.get('selected') else "" + res_std = summary.get('res_std', np.nan) + res_std_text = "n/a" if not np.isfinite(res_std) else f"{res_std * 100.0:.3f}%" + ax_lc.set_title(f"{targ_name} vs {summary['label']}{selected_text}\n{method_label} | scatter={res_std_text}") + ax_res.set_title("") + + label_slug = summary['label'].replace(" ", "") + fig.savefig(temp_dir / _dated_plot_filename( + "CompStarLightCurveFit", + label_slug, + targ_name, + date=date, + extension="png", + ), bbox_inches="tight") + fig.savefig(temp_dir / _dated_plot_filename( + "CompStarLightCurveFit", + label_slug, + targ_name, + date=date, + extension="pdf", + ), bbox_inches="tight") + plt.close(fig) + + +def _callable_accepts_keyword(callable_object, keyword): + try: + signature = inspect.signature(callable_object) + except (TypeError, ValueError): + return False + if keyword in signature.parameters: + return True + return any( + parameter.kind == inspect.Parameter.VAR_KEYWORD + for parameter in signature.parameters.values() + ) + + +def _plot_bestfit_for_lightcurve_png(fit, **requested_kwargs): + plotter = fit.plot_bestfit + plot_kwargs = { + key: value + for key, value in requested_kwargs.items() + if _callable_accepts_keyword(plotter, key) + } + return plotter(**plot_kwargs) + + +def _draw_comp_star_calibration_axis(axis, times, summary, colors): + axis.axhline(1.0, color='lightgray', lw=1.0, zorder=1) + ensemble_keep_mask = np.asarray(summary.get('ensemble_frame_keep_mask'), dtype=bool) + has_ensemble_keep_mask = ensemble_keep_mask.shape == times.shape + pairwise_series = summary.get('pairwise_ratio_series', {}) + for color_index, (other_label, ratio_series) in enumerate(pairwise_series.items()): + ratio_series = np.asarray(ratio_series, dtype=float) + line_ratio = ratio_series.copy() + if has_ensemble_keep_mask and line_ratio.shape == times.shape: + line_ratio[~ensemble_keep_mask] = np.nan + valid_time = np.isfinite(times) + valid_line = valid_time & np.isfinite(line_ratio) + if np.any(valid_line): + axis.plot(times[valid_time], line_ratio[valid_time], color=colors[color_index % len(colors)], + alpha=0.55, lw=1.0, label=other_label) + + ensemble_ratio = np.asarray(summary.get('ensemble_ratio_series'), dtype=float) + ensemble_time_valid = np.isfinite(times) + ensemble_valid = ensemble_time_valid & np.isfinite(ensemble_ratio) + if np.any(ensemble_valid): + line_ratio = ensemble_ratio.copy() + if has_ensemble_keep_mask and line_ratio.shape == times.shape: + line_ratio[~ensemble_keep_mask] = np.nan + line_valid = ensemble_time_valid & np.isfinite(line_ratio) + if np.any(line_valid): + axis.plot(times[ensemble_time_valid], line_ratio[ensemble_time_valid], color='black', lw=1.8, + label='Intercomparison') + if has_ensemble_keep_mask: + rejected = ensemble_valid & ~ensemble_keep_mask + if np.any(rejected): + axis.scatter(times[rejected], ensemble_ratio[rejected], marker='x', s=42, + color='red', linewidths=1.4, label='Intercomparison clip') + + selected_text = " selected" if summary.get('selected') else "" + aggregate = summary.get('aggregate_score', np.nan) + aggregate_text = "n/a" if not np.isfinite(aggregate) else f"{aggregate * 100.0:.3f}%" + axis.set_ylabel("Norm Ratio") + axis.set_title(f"{summary['label']}{selected_text} | suitability={aggregate_text}", loc='left', fontsize=10) + axis.grid(alpha=0.2) + handles, labels = axis.get_legend_handles_labels() + if handles and labels: + axis.legend(ncol=4, fontsize=8, loc='upper right') + + +def plot_comp_star_suitability(comp_summaries, targ_name, save, date, method_label): + if not comp_summaries: + return + + temp_dir = _working_artifacts_dir(save) + + labels = [summary['label'] for summary in comp_summaries] + positions = np.arange(len(labels)) + aggregate = np.array([summary.get('aggregate_score', np.nan) for summary in comp_summaries], dtype=float) * 100.0 + ensemble = np.array([summary.get('ensemble_score', np.nan) for summary in comp_summaries], dtype=float) * 100.0 + pairwise = np.array([summary.get('pairwise_median_score', np.nan) for summary in comp_summaries], dtype=float) * 100.0 + + fig, ax = plt.subplots(figsize=(max(7, 1.5 * len(labels)), 5)) + width = 0.25 + ax.bar(positions - width, aggregate, width=width, label='Suitability') + ax.bar(positions, ensemble, width=width, label='Intercomparison') + ax.bar(positions + width, pairwise, width=width, label='Pairwise median') + + for position, summary in zip(positions, comp_summaries): + if summary.get('selected'): + ax.text(position - width, aggregate[position] if np.isfinite(aggregate[position]) else 0.0, 'selected', + rotation=90, va='bottom', ha='center', fontsize=8, color='tomato') + + ax.set_xticks(positions) + ax.set_xticklabels(labels) + ax.set_ylabel("Residual Scatter [%]") + ax.set_title(f"{targ_name} Comparison-Star Suitability Summary\n{method_label}") + ax.legend() + ax.grid(axis='y', alpha=0.25) + fig.tight_layout() + fig.savefig(temp_dir / _dated_plot_filename("CompStarSuitability", targ_name, date=date, extension="png"), bbox_inches="tight") + fig.savefig(temp_dir / _dated_plot_filename("CompStarSuitability", targ_name, date=date, extension="pdf"), bbox_inches="tight") + plt.close(fig) + + +def plot_adaptive_aperture_diagnostics(times, aperture_series, annulus_series, fwhm_series, airmass, + targ_name, save, date, aperture_sigma, annulus_sigma): + times = np.asarray(times, dtype=float) + aperture_series = np.asarray(aperture_series, dtype=float) + annulus_series = np.asarray(annulus_series, dtype=float) + fwhm_series = np.asarray(fwhm_series, dtype=float) + airmass = np.asarray(airmass, dtype=float) + + plot_len = min(times.size, aperture_series.size, annulus_series.size, fwhm_series.size, airmass.size) + if plot_len == 0: + return + + times = times[:plot_len] + aperture_series = aperture_series[:plot_len] + annulus_series = annulus_series[:plot_len] + fwhm_series = fwhm_series[:plot_len] + airmass = airmass[:plot_len] + + valid_time = np.isfinite(times) + valid_aperture = np.isfinite(aperture_series) + valid_annulus = np.isfinite(annulus_series) + valid_fwhm = np.isfinite(fwhm_series) + valid_airmass = np.isfinite(airmass) + + temp_dir = _working_artifacts_dir(save) + + fig, axes = plt.subplots(2, 2, figsize=(12, 8.5)) + fig.suptitle( + f"{targ_name} Adaptive Aperture Diagnostics\n" + f"aper={aperture_sigma:.2f} sigma, annulus={annulus_sigma:.2f} sigma" + ) + + time_mask = valid_time & valid_aperture + time_zero = np.nanmin(times[time_mask]) if np.any(time_mask) else 0.0 + axes[0, 0].set_title("Aperture Radius vs Time") + axes[0, 0].set_xlabel(f"Time [BJD_TDB-{time_zero:.5f}]") + axes[0, 0].set_ylabel("Aperture Radius [px]") + if np.any(time_mask): + axes[0, 0].plot(times[time_mask] - time_zero, aperture_series[time_mask], color='tab:blue', + marker='o', ms=3, lw=1.1) + axes[0, 0].grid(alpha=0.25) + + annulus_mask = valid_time & valid_annulus + axes[0, 1].set_title("Annulus Width vs Time") + axes[0, 1].set_xlabel(f"Time [BJD_TDB-{time_zero:.5f}]") + axes[0, 1].set_ylabel("Annulus Width [px]") + if np.any(annulus_mask): + axes[0, 1].plot(times[annulus_mask] - time_zero, annulus_series[annulus_mask], color='tab:orange', + marker='o', ms=3, lw=1.1) + axes[0, 1].grid(alpha=0.25) + + fwhm_mask = valid_aperture & valid_fwhm + axes[1, 0].set_title("Aperture Radius vs Target FWHM") + axes[1, 0].set_xlabel("Target PSF FWHM [px]") + axes[1, 0].set_ylabel("Aperture Radius [px]") + if np.any(fwhm_mask): + axes[1, 0].scatter(fwhm_series[fwhm_mask], aperture_series[fwhm_mask], color='tab:green', s=18, alpha=0.8) + order = np.argsort(fwhm_series[fwhm_mask]) + axes[1, 0].plot(fwhm_series[fwhm_mask][order], aperture_series[fwhm_mask][order], color='tab:green', + alpha=0.35, lw=1.0) + axes[1, 0].grid(alpha=0.25) + + airmass_mask = valid_aperture & valid_airmass + axes[1, 1].set_title("Aperture Radius vs Airmass") + axes[1, 1].set_xlabel("Airmass") + axes[1, 1].set_ylabel("Aperture Radius [px]") + if np.any(airmass_mask): + axes[1, 1].scatter(airmass[airmass_mask], aperture_series[airmass_mask], color='tab:red', s=18, alpha=0.8) + order = np.argsort(airmass[airmass_mask]) + axes[1, 1].plot(airmass[airmass_mask][order], aperture_series[airmass_mask][order], color='tab:red', + alpha=0.35, lw=1.0) + axes[1, 1].grid(alpha=0.25) + + fig.tight_layout() + fig.savefig(temp_dir / _dated_plot_filename("AdaptiveApertureDiagnostics", targ_name, date=date, extension="png"), bbox_inches="tight") + fig.savefig(temp_dir / _dated_plot_filename("AdaptiveApertureDiagnostics", targ_name, date=date, extension="pdf"), bbox_inches="tight") + plt.close(fig) + + def plot_variable_residuals(save): plt.title("Stellar Variability Residuals") plt.ylabel("Residuals (flux)") plt.xlabel("Time [JD]") plt.legend() - plt.savefig(Path(save) / "temp" / f"Variable_Residuals.png") + plt.savefig(_working_artifacts_dir(save) / "Variable_Residuals.png") plt.close() +def _finite_plot_float(value): + try: + parsed = float(value) + except (TypeError, ValueError): + return None + return parsed if np.isfinite(parsed) else None + + +def _stellar_variability_reference_label(vsp_param, comparison_label): + comp_ra = _finite_plot_float(vsp_param.get('comp_ra')) + comp_dec = _finite_plot_float(vsp_param.get('comp_dec')) + details = [] + comparison_label = str(comparison_label).strip() if comparison_label else "" + + if comparison_label and not comparison_label.lower().startswith("ra="): + details.append(f"Label: {comparison_label}") + + if comp_ra is not None and comp_dec is not None: + details.extend((f"Comparison RA={comp_ra:.6f}", f"Dec={comp_dec:.6f}")) + + if not details and comparison_label: + details.append(comparison_label) + + return "\n".join(details) + + +def _stellar_variability_comparison_metadata_label(vsp_param): + details = [] + observed_filter = vsp_param.get('observed_filter') + if observed_filter: + details.append(f"Original filter: {observed_filter}") + + comparison_mag = magnitude_text( + vsp_param.get('catalog_mag_band') or vsp_param.get('mag_band') or 'V', + vsp_param.get('cmag'), + vsp_param.get('cmag_err'), + ) + if comparison_mag is not None: + details.append(f"Comparison mag: {comparison_mag}") + + return " | ".join(details) + + def plot_stellar_variability(vsp_params, save, s_name, vsp_auid_comp): + if not vsp_params: + return + + fig, ax = plt.subplots(figsize=(8, 5)) + plotted_points = 0 for vsp_p in vsp_params: - plt.errorbar(vsp_p['time'], vsp_p['mag'], yerr=vsp_p['mag_err'], color="tomato", fmt='.') + if not is_usable_apparent_magnitude(vsp_p.get('mag')): + continue + mag_err = normalized_magnitude_error(vsp_p.get('mag_err')) + ax.errorbar(vsp_p['time'], vsp_p['mag'], yerr=mag_err, color="tomato", fmt='.') + plotted_points += 1 + + if plotted_points == 0: + plt.close(fig) + return + + first_param = vsp_params[0] + band = first_param.get('mag_band') or 'V' + reference_label = _stellar_variability_reference_label(first_param, vsp_auid_comp) + title_lines = [s_name] + if reference_label: + title_lines.append(reference_label) + metadata_label = _stellar_variability_comparison_metadata_label(first_param) + if metadata_label: + title_lines.append(metadata_label) + ax.set_title("\n".join(title_lines), fontsize=11) + ax.set_ylabel(f"Magnitude ({band})") + ax.invert_yaxis() + ax.set_xlabel("Time [JD]") + fig.tight_layout() + output_dir = _working_artifacts_dir(save) + output_path = Path(save) / "Stellar_Variability.png" + fig.savefig(output_dir / "Stellar_Variability.png", bbox_inches="tight") + fig.savefig(output_path, bbox_inches="tight") + plt.close(fig) + + +def _stellar_variability_magnitude_series(vsp_params): + rows = [] + for vsp_p in vsp_params or []: + time_value = _finite_plot_float(vsp_p.get('time')) + mag_value = _finite_plot_float(vsp_p.get('mag')) + mag_err = normalized_magnitude_error(vsp_p.get('mag_err')) + if ( + time_value is None + or mag_value is None + or mag_err is None + or not is_usable_apparent_magnitude(mag_value) + ): + continue + rows.append((time_value, mag_value, mag_err, vsp_p)) + + if not rows: + return None + + rows.sort(key=lambda row: row[0]) + times = np.array([row[0] for row in rows], dtype=float) + magnitudes = np.array([row[1] for row in rows], dtype=float) + magnitude_errors = np.array([row[2] for row in rows], dtype=float) + return times, magnitudes, magnitude_errors, rows[0][3] - plt.title(f"{s_name} (Label: {vsp_auid_comp})") - plt.ylabel("Vmag") - plt.xlabel("Time [JD]") - plt.savefig(Path(save) / "temp" / f"Stellar_Variability.png") - plt.close() +def _stellar_variability_apparent_magnitude_calibration(fit): + series = _stellar_variability_magnitude_series( + getattr(fit, 'stellar_variability_params', None) + ) + if series is None: + return None + _, magnitudes, _, first_param = series + finite = np.isfinite(magnitudes) + if not np.any(finite): + return None + return { + 'baseline_magnitude': float(np.nanmedian(magnitudes[finite])), + 'band': first_param.get('mag_band') or 'V', + } + + +def _add_apparent_magnitude_axis(ax_lc, fit): + calibration = _stellar_variability_apparent_magnitude_calibration(fit) + if calibration is None: + return False + baseline_magnitude = calibration['baseline_magnitude'] + + def flux_to_magnitude(flux): + flux = np.asarray(flux, dtype=float) + with np.errstate(divide='ignore', invalid='ignore'): + return baseline_magnitude - (2.5 * np.log10(flux)) + + def magnitude_to_flux(magnitude): + magnitude = np.asarray(magnitude, dtype=float) + with np.errstate(over='ignore', invalid='ignore'): + return 10 ** ((baseline_magnitude - magnitude) / 2.5) + + secondary_axis = ax_lc.secondary_yaxis( + 'right', + functions=(flux_to_magnitude, magnitude_to_flux), + ) + secondary_axis.set_ylabel(f"Apparent Magnitude ({calibration['band']})") + return True + + +def plot_differential_magnitude(fit, target_name, save, date, observed_filter=None, + out_of_transit_only=False, apply_airmass_correction=None, + filename_prefix='DifferentialMagnitude', + save_stellar_variability_alias=False): + series = differential_magnitude_series_from_fit( + fit, + out_of_transit_only=out_of_transit_only, + apply_airmass_correction=apply_airmass_correction, + ) + if series is None: + return None + + fig, ax = plt.subplots(figsize=(8, 5)) + finite_error = np.isfinite(series['magnitude_error']) & (series['magnitude_error'] >= 0) + if np.any(finite_error): + ax.errorbar( + series['time'][finite_error], + series['magnitude'][finite_error], + yerr=series['magnitude_error'][finite_error], + color='royalblue', + fmt='.', + ) + if np.any(~finite_error): + ax.plot( + series['time'][~finite_error], + series['magnitude'][~finite_error], + '.', + color='royalblue', + ) + if getattr(fit, 'stellar_variability_only', False) or save_stellar_variability_alias: + ax.set_title(target_name) + else: + correction_label = ( + 'Airmass-corrected target/reference ratio' + if series['airmass_corrected'] + else 'Raw target/reference ratio' + ) + ax.set_title(f"{target_name}\n{correction_label}") + band_label = f" ({observed_filter})" if observed_filter else '' + ax.set_ylabel(f"Differential Magnitude{band_label}") + ax.invert_yaxis() + ax.set_xlabel("Time [BJD_TDB]") + fig.tight_layout() + + output_dir = Path(save) + output_dir.mkdir(parents=True, exist_ok=True) + png_path = output_dir / _dated_plot_filename( + filename_prefix, + target_name, + date=date, + extension='png', + ) + pdf_path = output_dir / _dated_plot_filename( + filename_prefix, + target_name, + date=date, + extension='pdf', + ) + fig.savefig(png_path, bbox_inches='tight') + fig.savefig(pdf_path, bbox_inches='tight') + + if save_stellar_variability_alias or getattr(fit, 'stellar_variability_only', False): + artifacts_dir = _working_artifacts_dir(save) + fig.savefig( + artifacts_dir / 'Stellar_Variability_DifferentialMagnitude.png', + bbox_inches='tight', + ) + fig.savefig( + output_dir / 'Stellar_Variability_DifferentialMagnitude.png', + bbox_inches='tight', + ) + plt.close(fig) + return png_path + + +# Observation statistics series selection +def _select_plot_rows(rows, sort_index=None, sigma_mask=None, relative_flux_mask=None): + rows = np.asarray(rows) + + if sort_index is not None: + rows = rows[np.asarray(sort_index)] + + if sigma_mask is not None: + sigma_mask = np.asarray(sigma_mask) + if sigma_mask.dtype == bool and rows.shape[0] == sigma_mask.shape[0]: + rows = rows[sigma_mask] + + if relative_flux_mask is not None: + relative_flux_mask = np.asarray(relative_flux_mask) + if relative_flux_mask.dtype == bool and rows.shape[0] == relative_flux_mask.shape[0]: + rows = rows[relative_flux_mask] + + return rows + + +def plot_obs_stats(fit, comp_stars, psf, si, gi, target_name, save, date, relative_flux_mask=None, + background_series=None): + fit_time = np.asarray(fit.time) + fit_airmass = np.asarray(fit.airmass) + temp_dir = _working_artifacts_dir(save) -# Observation statistics from PSF data -def plot_obs_stats(fit, comp_stars, psf, si, gi, target_name, save, date): for i in range(len(comp_stars) + 1): if i == 0: title, key = target_name, "target" @@ -195,45 +868,1019 @@ def plot_obs_stats(fit, comp_stars, psf, si, gi, target_name, save, date): fig, axs = plt.subplots(3, 2, figsize=(12, 10)) fig.suptitle(f"Observing Statistics - {title} - {date}") + star_stats = _select_plot_rows(psf[key], sort_index=si, sigma_mask=gi, + relative_flux_mask=relative_flux_mask) + background_data = None + if background_series is not None and key in background_series: + background_data = _select_plot_rows( + background_series[key], + sort_index=si, + sigma_mask=gi, + relative_flux_mask=relative_flux_mask, + ) + + plot_len_inputs = [fit_time.shape[0], fit_airmass.shape[0], star_stats.shape[0]] + if background_data is not None: + plot_len_inputs.append(background_data.shape[0]) + plot_len = min(plot_len_inputs) + if plot_len == 0: + plt.close(fig) + continue + + time_data = fit_time[:plot_len] + airmass_data = fit_airmass[:plot_len] + star_stats = star_stats[:plot_len] + if background_data is None: + background_data = star_stats[:, 6] + else: + background_data = np.asarray(background_data)[:plot_len] + axs[0, 0].set(xlabel="Time [BJD_TDB]", ylabel="X-Centroid [px]") - axs[0, 0].plot(fit.time, psf[key][si, 0][gi], 'k.') + axs[0, 0].plot(time_data, star_stats[:, 0], 'k.') axs[0, 1].set(xlabel="Time [BJD_TDB]", ylabel="Y-Centroid [px]") - axs[0, 1].plot(fit.time, psf[key][si, 1][gi], 'k.') + axs[0, 1].plot(time_data, star_stats[:, 1], 'k.') axs[1, 0].set(xlabel="Time [BJD_TDB]", ylabel="Seeing [px]") - axs[1, 0].plot(fit.time, 2.355 * 0.5 * (psf[key][si, 3][gi] + psf[key][si, 4][gi]), 'k.') + axs[1, 0].plot(time_data, 2.355 * 0.5 * (star_stats[:, 3] + star_stats[:, 4]), 'k.') axs[1, 1].set(xlabel="Time [BJD_TDB]", ylabel="Airmass") - axs[1, 1].plot(fit.time, fit.airmass, 'k.') + axs[1, 1].plot(time_data, airmass_data, 'k.') axs[2, 0].set(xlabel="Time [BJD_TDB]", ylabel="Amplitude [ADU]") - axs[2, 0].plot(fit.time, psf[key][si, 2][gi], 'k.') + axs[2, 0].plot(time_data, star_stats[:, 2], 'k.') axs[2, 1].set(xlabel="Time [BJD_TDB]", ylabel="Background [ADU]") - axs[2, 1].plot(fit.time, psf[key][si, 6][gi], 'k.') + axs[2, 1].plot(time_data, background_data, 'k.') plt.tight_layout() try: - fig.savefig(Path(save) / "temp" / f"Observing_Statistics_{key}_{date}.png", bbox_inches="tight") - fig.savefig(Path(save) / "temp" / f"Observing_Statistics_{key}_{date}.pdf", bbox_inches="tight") + fig.savefig(temp_dir / _dated_plot_filename("Observing_Statistics", key, date=date, extension="png"), bbox_inches="tight") + fig.savefig(temp_dir / _dated_plot_filename("Observing_Statistics", key, date=date, extension="pdf"), bbox_inches="tight") except Exception: pass plt.close() # Plotting Final Lightcurve -def plot_final_lightcurve(fit, high_res, targ_name, save, date): - f, (ax_lc, ax_res) = fit.plot_bestfit() +def _final_lightcurve_model_grid(fit, high_res): + if hasattr(fit, 'phase_upsample') and hasattr(fit, 'transit_upsample'): + x_values = np.asarray(fit.phase_upsample, dtype=float) + model = np.asarray(fit.transit_upsample, dtype=float) + times = getattr(fit, 'time_upsample', None) + if times is not None: + times = np.asarray(times, dtype=float) + if times.shape != model.shape: + times = None + return x_values, model, times + + phase = np.asarray(getattr(fit, 'phase', np.array([])), dtype=float) + model = np.asarray(high_res, dtype=float) + if phase.size == 0 or model.size == 0: + return None, None, None + x_values = np.linspace(np.nanmin(phase), np.nanmax(phase), model.size) + return x_values, model, None + + +def _transit_model_uncertainty_envelope_for_grid(fit, times, model_shape): + if times is None or times.shape != model_shape: + return None + + uncertainty_func = getattr(fit, 'transit_model_uncertainty', None) + if not callable(uncertainty_func): + return None + + try: + envelope = uncertainty_func(times) + except Exception: + return None + if envelope is None or len(envelope) != 2: + return None + + lower = np.asarray(envelope[0], dtype=float) + upper = np.asarray(envelope[1], dtype=float) + if lower.shape != model_shape or upper.shape != model_shape: + return None + return lower, upper + + +def _plot_final_data_scatter_uncertainty_band(ax_lc, fit, high_res): + empirical_uncertainty = getattr(fit, 'empirical_transit_uncertainty', None) + if not isinstance(empirical_uncertainty, dict) or not empirical_uncertainty.get('available'): + empirical_uncertainty = fit_empirical_transit_uncertainty(fit) + if not isinstance(empirical_uncertainty, dict) or not empirical_uncertainty.get('available'): + return False + + depth_uncertainty = empirical_uncertainty.get('depth_uncertainty_fraction') + try: + depth_uncertainty = float(depth_uncertainty) + except (TypeError, ValueError): + return False + if not np.isfinite(depth_uncertainty) or depth_uncertainty <= 0: + return False + + x_values, model, times = _final_lightcurve_model_grid(fit, high_res) + if x_values is None or model is None or x_values.shape != model.shape: + return False + + finite = np.isfinite(x_values) & np.isfinite(model) + if not np.any(finite): + return False + + empirical_lower = model - depth_uncertainty + empirical_upper = model + depth_uncertainty + sort_index = np.argsort(x_values) + x_sorted = x_values[sort_index] + finite_sorted = finite[sort_index] + empirical_lower_sorted = empirical_lower[sort_index] + empirical_upper_sorted = empirical_upper[sort_index] + + model_envelope = _transit_model_uncertainty_envelope_for_grid(fit, times, model.shape) + drew_band = False + def next_label(): + nonlocal drew_band + drew_band = True + return '_nolegend_' + + if model_envelope is not None: + model_lower, model_upper = model_envelope + model_lower_sorted = np.asarray(model_lower, dtype=float)[sort_index] + model_upper_sorted = np.asarray(model_upper, dtype=float)[sort_index] + + upper_region = ( + finite_sorted + & np.isfinite(empirical_upper_sorted) + & np.isfinite(model_upper_sorted) + & (empirical_upper_sorted > model_upper_sorted) + ) + lower_region = ( + finite_sorted + & np.isfinite(empirical_lower_sorted) + & np.isfinite(model_lower_sorted) + & (empirical_lower_sorted < model_lower_sorted) + ) + if np.any(upper_region): + ax_lc.fill_between( + x_sorted, + model_upper_sorted, + empirical_upper_sorted, + where=upper_region, + interpolate=True, + color='#6a1b9a', + alpha=0.16, + linewidth=0, + zorder=2.35, + label=next_label(), + ) + if np.any(lower_region): + ax_lc.fill_between( + x_sorted, + empirical_lower_sorted, + model_lower_sorted, + where=lower_region, + interpolate=True, + color='#6a1b9a', + alpha=0.16, + linewidth=0, + zorder=2.35, + label=next_label(), + ) + return drew_band + + ax_lc.fill_between( + x_sorted, + empirical_lower_sorted, + empirical_upper_sorted, + where=finite_sorted, + interpolate=True, + color='#6a1b9a', + alpha=0.14, + linewidth=0, + zorder=2.0, + label=next_label(), + ) + return drew_band + + +def _plot_final_residual_rejected_points(ax_lc, ax_res, fit): + rejection = getattr(fit, 'final_residual_rejection', None) + if not isinstance(rejection, dict) or not rejection.get('applied'): + return + + phase = np.asarray(rejection.get('rejected_phase', []), dtype=float) + flux = np.asarray(rejection.get('rejected_flux', []), dtype=float) + residual_percent = np.asarray(rejection.get('rejected_residual_percent', []), dtype=float) + plot_count = min(phase.size, flux.size, residual_percent.size) + if plot_count == 0: + return + + phase = phase[:plot_count] + flux = flux[:plot_count] + residual_percent = residual_percent[:plot_count] + finite = np.isfinite(phase) & np.isfinite(flux) & np.isfinite(residual_percent) + if not np.any(finite): + return + + ax_lc.scatter( + phase[finite], + flux[finite], + marker='x', + s=58, + linewidths=1.6, + color='red', + zorder=1200, + label='_nolegend_', + ) + ax_res.scatter( + phase[finite], + residual_percent[finite], + marker='x', + s=58, + linewidths=1.6, + color='red', + zorder=1200, + label='_nolegend_', + ) + + +def plot_final_lightcurve(fit, high_res, targ_name, save, date, observed_filter=None): + plot_differential_magnitude( + fit, + getattr(fit, 'stellar_variability_target_name', targ_name), + save, + date, + observed_filter=observed_filter, + ) + fit_shape = np.asarray(getattr(fit, 'data', []), dtype=float).shape + has_raw_stellar_photometry = ( + np.asarray( + getattr(fit, 'stellar_variability_target_flux', []), + dtype=float, + ).shape == fit_shape + and np.asarray( + getattr(fit, 'stellar_variability_comp_flux', []), + dtype=float, + ).shape == fit_shape + ) + if not getattr(fit, 'stellar_variability_only', False) and has_raw_stellar_photometry: + plot_differential_magnitude( + fit, + getattr(fit, 'stellar_variability_target_name', targ_name), + save, + date, + observed_filter=observed_filter, + out_of_transit_only=True, + apply_airmass_correction=False, + filename_prefix='StellarVariabilityDifferentialMagnitude', + save_stellar_variability_alias=True, + ) + if getattr(fit, 'stellar_variability_only', False): + series = _stellar_variability_magnitude_series( + getattr(fit, 'stellar_variability_params', None) + ) + if series is None: + return + + obs_time, magnitudes, magnitude_errors, first_param = series + f, ax_lc = plt.subplots(figsize=(8, 5)) + title_name = getattr(fit, 'stellar_variability_target_name', targ_name) + title_lines = [title_name] + reference_label = _stellar_variability_reference_label( + first_param, + getattr(fit, 'stellar_variability_reference_label', first_param.get('cname')), + ) + if reference_label: + title_lines.append(reference_label) + metadata_label = _stellar_variability_comparison_metadata_label(first_param) + if metadata_label: + title_lines.append(metadata_label) + + ax_lc.set_title("\n".join(title_lines), fontsize=11) + ax_lc.errorbar( + obs_time, + magnitudes, + yerr=magnitude_errors, + color="tomato", + fmt='.', + ) + band = first_param.get('mag_band') or 'V' + ax_lc.set_ylabel(f"Magnitude ({band})") + ax_lc.invert_yaxis() + ax_lc.set_xlabel("Time [BJD_TDB]") + f.tight_layout() + + Path(save).mkdir(parents=True, exist_ok=True) + try: + f.savefig(Path(save) / _dated_plot_filename("FinalLightCurve", targ_name, date=date, extension="png"), bbox_inches="tight") + f.savefig(Path(save) / _dated_plot_filename("FinalLightCurve", targ_name, date=date, extension="pdf"), bbox_inches="tight") + except Exception: + pass + plt.close(f) + return + + empirical_uncertainty = getattr(fit, 'empirical_transit_uncertainty', None) + if not isinstance(empirical_uncertainty, dict) or not empirical_uncertainty.get('available'): + empirical_uncertainty = fit_empirical_transit_uncertainty(fit) + if isinstance(empirical_uncertainty, dict) and empirical_uncertainty.get('available'): + try: + fit.empirical_transit_uncertainty = empirical_uncertainty + except Exception: + pass + + f, (ax_lc, ax_res) = _plot_bestfit_for_lightcurve_png( + fit, + show_flux_baseline_label=False, + show_model_uncertainty=True, + show_baseline_uncertainty=True, + ) ax_lc.set_title(targ_name) - ax_lc.plot(np.linspace(np.nanmin(fit.phase), np.nanmax(fit.phase), 1000), high_res, 'r', zorder=1000, lw=2) + drew_data_scatter_band = _plot_final_data_scatter_uncertainty_band(ax_lc, fit, high_res) + if hasattr(fit, 'phase_upsample') and hasattr(fit, 'transit_upsample'): + ax_lc.plot(fit.phase_upsample, fit.transit_upsample, 'r', zorder=1000, lw=2) + else: + ax_lc.plot(np.linspace(np.nanmin(fit.phase), np.nanmax(fit.phase), 1000), high_res, 'r', zorder=1000, lw=2) + _plot_final_residual_rejected_points(ax_lc, ax_res, fit) + if drew_data_scatter_band: + ax_lc.legend(loc='best') + _add_apparent_magnitude_axis(ax_lc, fit) Path(save).mkdir(parents=True, exist_ok=True) try: - f.savefig(Path(save) / f"FinalLightCurve_{targ_name}_{date}.png", bbox_inches="tight") - f.savefig(Path(save) / f"FinalLightCurve_{targ_name}_{date}.pdf", bbox_inches="tight") + f.savefig(Path(save) / _dated_plot_filename("FinalLightCurve", targ_name, date=date, extension="png"), bbox_inches="tight") + f.savefig(Path(save) / _dated_plot_filename("FinalLightCurve", targ_name, date=date, extension="pdf"), bbox_inches="tight") except Exception: pass plt.close() + + +def _plot_scalar(value, default=np.nan): + try: + result = np.asarray(value, dtype=float).reshape(-1) + except (TypeError, ValueError): + return default + if result.size == 0: + return default + result = float(result[0]) + return result if np.isfinite(result) else default + + +def _plot_positive_error(value): + value = _plot_scalar(value) + if not np.isfinite(value) or value < 0: + return np.nan + return value + + +def _format_parameter_value(value, error=None, unit="", split_error=False): + value = _plot_scalar(value) + if not np.isfinite(value): + return "n/a" + + suffix = f" {unit}" if unit else "" + if error is None: + return f"{value:.6f}".rstrip('0').rstrip('.') + suffix + + error = _plot_positive_error(error) + if np.isfinite(error): + formatted = format_value_with_uncertainty(value, error) + if split_error: + formatted = formatted.replace(" +/- ", "\n+/- ", 1) + return f"{formatted}{suffix}" + return f"{value:.6f}".rstrip('0').rstrip('.') + suffix + + +def _prior_impact_parameter_value_error(planet_dict): + ars = _plot_scalar(planet_dict.get('aRs')) + inc = _plot_scalar(planet_dict.get('inc')) + if not np.isfinite(ars) or not np.isfinite(inc): + return np.nan, np.nan + + ecc = _plot_scalar(planet_dict.get('ecc'), 0.0) + omega = np.deg2rad(_plot_scalar(planet_dict.get('omega'), 0.0)) + denominator = 1.0 + ecc * np.sin(omega) + if not np.isfinite(denominator) or np.isclose(denominator, 0.0): + return np.nan, np.nan + + scale_factor = (1.0 - ecc ** 2) / denominator + inc_rad = np.deg2rad(inc) + impact_parameter = scale_factor * ars * np.cos(inc_rad) + + ars_error = _plot_positive_error(planet_dict.get('aRsUnc')) + inc_error = _plot_positive_error(planet_dict.get('incUnc')) + if np.isfinite(ars_error) and np.isfinite(inc_error): + impact_error = np.hypot( + scale_factor * np.cos(inc_rad) * ars_error, + scale_factor * ars * np.sin(inc_rad) * np.deg2rad(inc_error), + ) + else: + impact_error = np.nan + + return float(impact_parameter), float(impact_error) if np.isfinite(impact_error) else np.nan + + +def _ephemeris_prior_at_posterior_epoch(planet_dict, posterior_tmid): + mid_t = _plot_scalar(planet_dict.get('midT')) + period = _plot_scalar(planet_dict.get('pPer')) + posterior_tmid = _plot_scalar(posterior_tmid) + if not np.isfinite(mid_t) or not np.isfinite(period) or period <= 0 or not np.isfinite(posterior_tmid): + return mid_t, _plot_positive_error(planet_dict.get('midTUnc')), None + + epoch = int(np.round((posterior_tmid - mid_t) / period)) + expected_tmid = mid_t + epoch * period + + error_terms = [] + mid_t_error = _plot_positive_error(planet_dict.get('midTUnc')) + period_error = _plot_positive_error(planet_dict.get('pPerUnc')) + if np.isfinite(mid_t_error): + error_terms.append(mid_t_error) + if np.isfinite(period_error): + error_terms.append(abs(epoch) * period_error) + + if error_terms: + expected_error = float(np.sqrt(np.sum(np.square(error_terms)))) + else: + expected_error = np.nan + return float(expected_tmid), expected_error, epoch + + +def _posterior_parameter_value_error(fit, parameter_key, empirical_uncertainty): + parameters = getattr(fit, 'parameters', {}) or {} + errors = getattr(fit, 'errors', {}) or {} + + if parameter_key == 'b': + errors_override = {} + errors = getattr(fit, 'errors', {}) or {} + sample_errors = getattr(fit, 'sample_errors', {}) or {} + b_error = _plot_positive_error(errors.get('b')) + if not np.isfinite(b_error): + b_error = _plot_positive_error(sample_errors.get('b')) + if np.isfinite(b_error): + errors_override['b'] = float(b_error * empirical_red_noise_error_scale(empirical_uncertainty)) + ars_error = fit_parameter_model_data_uncertainty( + fit, + 'ars', + empirical_uncertainty=empirical_uncertainty, + ) + inc_error = fit_parameter_model_data_uncertainty( + fit, + 'inc', + empirical_uncertainty=empirical_uncertainty, + ) + if np.isfinite(ars_error): + errors_override['ars'] = ars_error + if np.isfinite(inc_error): + errors_override['inc'] = inc_error + return fit_impact_parameter_value_error(fit, errors_override=errors_override) + + value = _plot_scalar(parameters.get(parameter_key)) + if parameter_key == 'rprs': + error = _plot_positive_error( + (empirical_uncertainty or {}).get('combined_rprs_uncertainty') + ) + if not np.isfinite(error): + error = _plot_positive_error(errors.get(parameter_key)) + return value, error + + error = fit_parameter_model_data_uncertainty( + fit, + parameter_key, + empirical_uncertainty=empirical_uncertainty, + ) + if not np.isfinite(error): + error = _plot_positive_error(errors.get(parameter_key)) + return value, error + + +def _prior_posterior_comparison_rows(fit, planet_dict): + empirical_uncertainty = getattr(fit, 'empirical_transit_uncertainty', None) + if not isinstance(empirical_uncertainty, dict) or not empirical_uncertainty.get('available'): + empirical_uncertainty = fit_empirical_transit_uncertainty(fit) + if isinstance(empirical_uncertainty, dict) and empirical_uncertainty.get('available'): + try: + fit.empirical_transit_uncertainty = empirical_uncertainty + except Exception: + pass + + definitions = [ + ("Tmid", "tmid", "midT", "midTUnc", "", True), + ("Rp/R*", "rprs", "rprs", "rprsUnc", "", False), + ("a/Rs", "ars", "aRs", "aRsUnc", "", False), + ("Inc.", "inc", "inc", "incUnc", "deg", False), + ("b", "b", None, None, "", False), + ] + + rows = [] + rprs_prior_fallback = bool( + getattr(fit, 'rprs_prior_fallback_applied', False) + or (isinstance(empirical_uncertainty, dict) + and empirical_uncertainty.get('rprs_prior_fallback_applied')) + or (isinstance(empirical_uncertainty, dict) + and empirical_uncertainty.get('rprs_uncertainty_basis') == 'prior_assumed_data_only') + ) + omitted_notes = [] + for label, parameter_key, prior_key, prior_error_key, unit, split_error in definitions: + if parameter_key == 'rprs' and rprs_prior_fallback: + prior_value = _plot_scalar(planet_dict.get(prior_key)) + prior_error = _plot_positive_error(planet_dict.get(prior_error_key)) + posterior_value, posterior_error = _posterior_parameter_value_error( + fit, + parameter_key, + empirical_uncertainty, + ) + omitted_notes.append( + "Rp/R* omitted: prior value assumed, not measured " + f"({_format_parameter_value(prior_value, prior_error)}; " + f"data-only uncertainty {_format_parameter_value(posterior_value, posterior_error)})." + ) + continue + + posterior_value, posterior_error = _posterior_parameter_value_error( + fit, + parameter_key, + empirical_uncertainty, + ) + + if parameter_key == 'b': + prior_value, prior_error = _prior_impact_parameter_value_error(planet_dict) + prior_label = "Prior" + elif parameter_key == 'tmid': + prior_value, prior_error, _ = _ephemeris_prior_at_posterior_epoch( + planet_dict, + posterior_value, + ) + prior_label = "Prior" + else: + prior_value = _plot_scalar(planet_dict.get(prior_key)) + prior_error = _plot_positive_error(planet_dict.get(prior_error_key)) + prior_label = "Prior" + + if not np.isfinite(prior_value) or not np.isfinite(posterior_value): + continue + + error_terms = [ + term for term in (prior_error, posterior_error) + if np.isfinite(term) and term > 0 + ] + if error_terms: + combined_sigma = float(np.sqrt(np.sum(np.square(error_terms)))) + else: + separation = abs(posterior_value - prior_value) + combined_sigma = float(separation) if separation > 0 else np.nan + if not np.isfinite(combined_sigma) or combined_sigma <= 0: + continue + + posterior_offset = (posterior_value - prior_value) / combined_sigma + prior_error_sigma = prior_error / combined_sigma if np.isfinite(prior_error) else 0.0 + posterior_error_sigma = ( + posterior_error / combined_sigma if np.isfinite(posterior_error) else 0.0 + ) + prior_assumed = parameter_key == 'rprs' and rprs_prior_fallback + + rows.append({ + "label": label, + "parameter_key": parameter_key, + "posterior_offset": float(posterior_offset), + "prior_error_sigma": float(prior_error_sigma), + "posterior_error_sigma": float(posterior_error_sigma), + "prior_text": _format_parameter_value( + prior_value, + prior_error, + unit=unit, + split_error=split_error, + ), + "posterior_text": _format_parameter_value( + posterior_value, + posterior_error, + unit=unit, + split_error=split_error, + ), + "prior_label": prior_label, + "prior_assumed": prior_assumed, + }) + + return rows, omitted_notes + + +def plot_prior_posterior_comparison(fit, planet_dict, targ_name, save, date): + rows, omitted_notes = _prior_posterior_comparison_rows(fit, planet_dict) + if not rows: + return None + + note_height = 0.34 * len(omitted_notes) + row_spacing = 1.35 + fig_height = max(5.0, 1.02 * len(rows) + 2.0 + note_height) + fig, ax = plt.subplots(figsize=(11.8, fig_height)) + + y_positions = np.arange(len(rows), dtype=float) * row_spacing + posterior_offsets = np.array([row["posterior_offset"] for row in rows], dtype=float) + prior_errors = np.array([row["prior_error_sigma"] for row in rows], dtype=float) + posterior_errors = np.array([row["posterior_error_sigma"] for row in rows], dtype=float) + + xmin = min(-3.5, np.nanmin(np.r_[posterior_offsets - posterior_errors, -prior_errors]) - 0.45) + xmax = max(3.5, np.nanmax(np.r_[posterior_offsets + posterior_errors, prior_errors]) + 0.45) + + ax.axvspan(-1.0, 1.0, color='#2e7d32', alpha=0.08, linewidth=0) + ax.axvspan(-3.0, 3.0, color='#f9a825', alpha=0.06, linewidth=0) + ax.axvline(0.0, color='0.25', lw=1.2, ls='--', zorder=1) + + ax.errorbar( + np.zeros_like(y_positions), + y_positions + 0.13, + xerr=prior_errors, + fmt='o', + ms=6, + color='#1565c0', + ecolor='#1565c0', + elinewidth=1.4, + capsize=3, + label='Prior', + zorder=5, + ) + ax.errorbar( + posterior_offsets, + y_positions - 0.13, + xerr=posterior_errors, + fmt='s', + ms=6, + color='#c62828', + ecolor='#c62828', + elinewidth=1.4, + capsize=3, + label='Posterior', + zorder=6, + ) + + for y_position, row in zip(y_positions, rows): + annotation = ( + f"{row['prior_label']}\n" + f"{row['prior_text']}\n" + "Posterior\n" + f"{row['posterior_text']}" + ) + ax.text( + 1.015, + y_position, + annotation, + transform=ax.get_yaxis_transform(), + ha='left', + va='center', + fontsize=8.5, + linespacing=1.12, + color='0.18', + ) + + ax.set_yticks(y_positions) + ax.set_yticklabels([row["label"] for row in rows]) + ax.invert_yaxis() + ax.set_xlim(xmin, xmax) + ax.set_xlabel("Posterior offset from prior [combined sigma]") + ax.set_title(f"{targ_name} Prior vs Posterior Transit Parameters") + ax.grid(axis='x', alpha=0.28) + if omitted_notes: + ax.text( + 0.0, + -0.16, + "\n".join(omitted_notes), + transform=ax.transAxes, + ha='left', + va='top', + fontsize=9, + color='0.22', + ) + ax.legend( + handles=[ + Line2D([0], [0], marker='o', color='none', markerfacecolor='#1565c0', + markeredgecolor='#1565c0', markersize=7, label='Prior'), + Line2D([0], [0], marker='s', color='none', markerfacecolor='#c62828', + markeredgecolor='#c62828', markersize=7, label='Posterior'), + ], + loc='lower right', + ) + fig.subplots_adjust(right=0.64) + + Path(save).mkdir(parents=True, exist_ok=True) + png_path = Path(save) / _dated_plot_filename( + "PriorPosteriorComparison", + targ_name, + date=date, + extension="png", + ) + pdf_path = Path(save) / _dated_plot_filename( + "PriorPosteriorComparison", + targ_name, + date=date, + extension="pdf", + ) + try: + fig.savefig(png_path, bbox_inches="tight") + fig.savefig(pdf_path, bbox_inches="tight") + except Exception: + png_path = None + plt.close(fig) + return png_path + + +def _fit_ktmf_metric_contributions_status(fit): + transit_qc = getattr(fit, 'transit_qc', None) + if not isinstance(transit_qc, dict): + transit_qc = {} + + metric = _plot_scalar( + getattr(fit, 'transit_qc_ktmf_metric', transit_qc.get('ktmf_metric', np.nan)) + ) + contributions = getattr(fit, 'transit_qc_ktmf_contributions', None) + if not contributions: + contributions = transit_qc.get('ktmf_contributions', []) + if not isinstance(contributions, (list, tuple)): + contributions = [] + + status = _ktmf_status_from_metric(metric) + if not status: + status = getattr(fit, 'transit_qc_status', transit_qc.get('status', None)) + return metric, list(contributions), status + + +def _ktmf_status_from_metric(metric): + metric = _plot_scalar(metric) + if not np.isfinite(metric): + return None + if metric >= 4.0: + return "pass" + if metric >= 3.0: + return "marginal" + return "fail" + + +def _short_ktmf_label(label): + replacements = { + "Deviation From Expected Value": "Expected Rp/R*", + "Residual Scatter Around Full Model Fit": "Residual scatter", + "Residual Flatness": "Residual flatness", + "Tmid Posterior Gaussianity": "Tmid Gaussianity", + "Duration Consistency": "Duration", + "EEBLS Depth SNR": "EEBLS SNR", + "Sampling / Cadence": "Sampling", + } + return replacements.get(str(label), str(label)) + + +def _ktmf_marker_color(score): + score = _plot_scalar(score) + if not np.isfinite(score): + return '0.45' + if score >= 0.8: + return '#2e7d32' + if score >= 0.6: + return '#f9a825' + return '#c62828' + + +def _format_ktmf_metric(value, maximum=5.0): + value = _plot_scalar(value) + maximum = _plot_scalar(maximum) + if not np.isfinite(value): + return "n/a" + if np.isfinite(maximum) and maximum > 0: + return f"{value:.2f} / {maximum:.2f}" + return f"{value:.2f}" + + +def _format_ktmf_component_annotation(row): + if row.get('kind') == 'total': + status = row.get('status') + status_text = f"\n{status.upper()}" if status else "" + uncertainty = _plot_positive_error(row.get('score_uncertainty')) + uncertainty_text = f"\nscore spread +/- {uncertainty:.2f}" if np.isfinite(uncertainty) else "" + return f"KTMF\n{_format_ktmf_metric(row.get('points'), row.get('max_points'))}{status_text}{uncertainty_text}" + + if not row.get('available', True): + return "Not scored" + + score_uncertainty = _plot_positive_error(row.get('score_uncertainty')) + if np.isfinite(score_uncertainty): + score_text = _format_parameter_value(row.get('score'), score_uncertainty) + else: + score_text = _format_parameter_value(row.get('score')) + return ( + f"Score\n{score_text}\n" + f"Points\n{_format_ktmf_metric(row.get('points'), row.get('max_points'))}" + ) + + +def _compact_ktmf_detail(row): + detail = row.get('detail') + if not detail: + return None + detail = str(detail) + if row.get('label') == "Expected Rp/R*": + if "fixed to the input prior" in detail or "prior" in detail.lower(): + return "Rp/R* prior assumed; not scored." + keep = [] + for part in detail.split(','): + part = part.strip() + if part.startswith("Rp/R* sigma="): + keep.append(part) + return ", ".join(keep) if keep else detail + return detail + + +def _ktmf_plot_rows(fit): + metric, contributions, status = _fit_ktmf_metric_contributions_status(fit) + component_rows = [] + for contribution in contributions: + if not isinstance(contribution, dict): + continue + available = bool(contribution.get('available', True)) + score = _plot_scalar(contribution.get('score')) + if not available or not np.isfinite(score): + score = np.nan + component_rows.append({ + "kind": "component", + "label": _short_ktmf_label(contribution.get('label', 'KTMF component')), + "score": float(np.clip(score, 0.0, 1.0)) if np.isfinite(score) else np.nan, + "score_uncertainty": _plot_positive_error(contribution.get('score_uncertainty')), + "points": _plot_scalar(contribution.get('points'), 0.0), + "max_points": _plot_scalar(contribution.get('max_points'), 0.0), + "available": available and np.isfinite(score), + "detail": contribution.get('detail'), + }) + + rows = [] + if np.isfinite(metric): + total_score = float(np.clip(metric / 5.0, 0.0, 1.0)) + available_component_rows = [ + row for row in component_rows + if row.get('available') + and np.isfinite(row.get('score', np.nan)) + and np.isfinite(row.get('max_points', np.nan)) + and row.get('max_points', 0.0) > 0 + ] + score_uncertainty = np.nan + if len(available_component_rows) > 1: + scores = np.asarray([row['score'] for row in available_component_rows], dtype=float) + weights = np.asarray([row['max_points'] for row in available_component_rows], dtype=float) + if np.isfinite(weights).all() and np.sum(weights) > 0: + score_uncertainty = float( + np.sqrt(np.average((scores - total_score) ** 2, weights=weights)) + ) + rows.append({ + "kind": "total", + "label": "KTMF total", + "score": total_score, + "score_uncertainty": score_uncertainty, + "points": float(metric), + "max_points": 5.0, + "available": True, + "status": status, + }) + rows.extend(component_rows) + return rows + + +def _score_errorbar_limits(score, uncertainty): + score = _plot_scalar(score) + uncertainty = _plot_positive_error(uncertainty) + if not np.isfinite(score) or not np.isfinite(uncertainty) or uncertainty <= 0: + return None + lower = min(uncertainty, max(score, 0.0)) + upper = min(uncertainty, max(1.0 - score, 0.0)) + if lower <= 0 and upper <= 0: + return None + return np.asarray([[lower], [upper]], dtype=float) + + +def _draw_ktmf_score_background(ax): + ax.axvspan(0.0, 0.6, color='#c62828', alpha=0.055, linewidth=0) + ax.axvspan(0.6, 0.8, color='#f9a825', alpha=0.09, linewidth=0) + ax.axvspan(0.8, 1.0, color='#2e7d32', alpha=0.08, linewidth=0) + ax.axvline(0.6, color='0.55', lw=1.0, ls=':', zorder=1) + ax.axvline(0.8, color='0.45', lw=1.1, ls='--', zorder=1) + ax.grid(axis='x', alpha=0.28) + + +def _plot_ktmf_score_marker(ax, row, y_position): + score = _plot_scalar(row.get('score')) + color = _ktmf_marker_color(score) + marker = 'D' if row.get('kind') == 'total' else 's' + marker_size = 62 if row.get('kind') == 'total' else 48 + marker_scale = 3.0 + if np.isfinite(score): + ax.errorbar( + [score], + [y_position], + xerr=_score_errorbar_limits(score, row.get('score_uncertainty')), + fmt=marker, + ms=np.sqrt(marker_size) * marker_scale, + color=color, + ecolor=color, + elinewidth=1.8, + capsize=4, + markeredgecolor='white', + markeredgewidth=1.2, + zorder=5, + ) + else: + ax.scatter( + [0.0], + [y_position], + marker='x', + s=52 * marker_scale ** 2, + color='0.45', + linewidths=2.0, + zorder=5, + ) + + +def plot_ktmf_qc_metrics(fit, targ_name, save, date): + rows = _ktmf_plot_rows(fit) + if not rows: + return None + + total_rows = [row for row in rows if row.get('kind') == 'total'] + component_rows = [row for row in rows if row.get('kind') != 'total'] + row_spacing = 1.35 + component_height = max(3.6, 0.98 * max(len(component_rows), 1) + 1.3) + fig_height = component_height + (1.55 if total_rows else 0.0) + if total_rows: + fig, (ax_total, ax_components) = plt.subplots( + 2, + 1, + figsize=(11.8, fig_height), + sharex=True, + gridspec_kw={'height_ratios': [1.0, component_height]}, + ) + axes = [ax_total, ax_components] + else: + fig, ax_components = plt.subplots(figsize=(11.8, fig_height)) + ax_total = None + axes = [ax_components] + + for axis in axes: + _draw_ktmf_score_background(axis) + axis.set_xlim(-0.05, 1.05) + + if total_rows: + total_row = total_rows[0] + _plot_ktmf_score_marker(ax_total, total_row, 0.0) + ax_total.text( + 1.025, + 0.0, + _format_ktmf_component_annotation(total_row), + transform=ax_total.get_yaxis_transform(), + ha='left', + va='center', + fontsize=8.5, + linespacing=1.12, + color='0.18', + ) + ax_total.set_yticks([0.0]) + ax_total.set_yticklabels([total_row["label"]]) + ax_total.set_ylim(0.65, -0.65) + ax_total.tick_params(axis='x', labelbottom=False) + ax_total.set_title(f"{targ_name} KTMF QC Metrics") + + component_positions = np.arange(len(component_rows), dtype=float) * row_spacing + for y_position, row in zip(component_positions, component_rows): + _plot_ktmf_score_marker(ax_components, row, y_position) + ax_components.text( + 1.025, + y_position, + _format_ktmf_component_annotation(row), + transform=ax_components.get_yaxis_transform(), + ha='left', + va='center', + fontsize=8.5, + linespacing=1.12, + color='0.18', + ) + + ax_components.set_yticks(component_positions) + ax_components.set_yticklabels([row["label"] for row in component_rows]) + ax_components.invert_yaxis() + ax_components.set_xlabel("KTMF component score fraction") + if not total_rows: + ax_components.set_title(f"{targ_name} KTMF QC Metrics") + fig.subplots_adjust(right=0.62, hspace=0.12) + + Path(save).mkdir(parents=True, exist_ok=True) + png_path = Path(save) / _dated_plot_filename( + "KTMF_QC", + targ_name, + date=date, + extension="png", + ) + pdf_path = Path(save) / _dated_plot_filename( + "KTMF_QC", + targ_name, + date=date, + extension="pdf", + ) + try: + fig.savefig(png_path, bbox_inches="tight") + fig.savefig(pdf_path, bbox_inches="tight") + except Exception: + png_path = None + plt.close(fig) + return png_path diff --git a/exotic/transit_depth.py b/exotic/transit_depth.py new file mode 100644 index 00000000..e1a55be9 --- /dev/null +++ b/exotic/transit_depth.py @@ -0,0 +1,438 @@ +import math + +import numpy as np + + +AREA_DEPTH_LABEL = "Radius-ratio area depth (Rp/R*)^2" +OBSERVABLE_DEPTH_LABEL = "Observable model transit depth" +PRIOR_OBSERVABLE_DEPTH_LABEL = "Prior observable model transit depth" +OBSERVABLE_DEPTH_DELTA_LABEL = "Observable model depth change from prior" + +_DEPTH_ERROR_KEYS = ("rprs", "ars", "inc", "ecc", "omega", "u0", "u1", "u2", "u3") +_REQUIRED_TRANSIT_KEYS = ("rprs", "per", "ars", "inc", "ecc", "omega", "tmid", "u0", "u1", "u2", "u3") + + +def finite_float(value, default=np.nan): + try: + value = float(value) + except (TypeError, ValueError): + return default + return value if np.isfinite(value) else default + + +def radius_ratio_area_depth_percent(rprs, rprs_error=None): + rprs = finite_float(rprs) + if not np.isfinite(rprs) or rprs < 0: + return np.nan, np.nan + + depth = 100.0 * rprs ** 2 + rprs_error = finite_float(rprs_error) + if np.isfinite(rprs_error) and rprs_error >= 0: + return float(depth), float(200.0 * abs(rprs) * rprs_error) + return float(depth), np.nan + + +def planet_dict_transit_parameters(planet_dict, limb_darkening=None, fallback=None): + values = dict(fallback or {}) + planet_dict = planet_dict or {} + + aliases = { + "rprs": ( + "rprs", + "pl_ratror", + "Rp/Rs", + "Rp/R*", + "Ratio of Planet to Stellar Radius (Rp/Rs)", + "Ratio of Planet to Stellar Radius (Rp/R*)", + ), + "per": ( + "pPer", + "pl_orbper", + "per", + "period", + "Orbital Period (days)", + ), + "ars": ( + "aRs", + "pl_ratdor", + "ars", + "a/Rs", + "a/R*", + "Ratio of Distance to Stellar Radius (a/Rs)", + "Ratio of Distance to Stellar Radius (a/R*)", + ), + "inc": ( + "inc", + "pl_orbincl", + "Orbital Inclination (deg)", + ), + "ecc": ( + "ecc", + "pl_orbeccen", + "Orbital Eccentricity", + "Orbital Eccentricity (0 if null)", + ), + "omega": ( + "omega", + "pl_orblper", + "Argument of Periastron (deg)", + ), + "tmid": ( + "midT", + "pl_tranmid", + "tmid", + "Published Mid-Transit Time", + "Published Mid-Transit Time (BJD-UTC)", + "Published Mid-Transit Time (BJD_UTC)", + ), + } + for target, names in aliases.items(): + for name in names: + if name not in planet_dict: + continue + value = finite_float(planet_dict.get(name)) + if np.isfinite(value): + values[target] = value + break + + if limb_darkening is not None: + for index, item in enumerate(limb_darkening): + if index > 3: + break + if isinstance(item, (list, tuple, np.ndarray)): + value = item[0] if len(item) else np.nan + else: + value = item + value = finite_float(value) + if np.isfinite(value): + values[f"u{index}"] = value + + for key in ("u0", "u1", "u2", "u3"): + values.setdefault(key, 0.0) + values.setdefault("ecc", 0.0) + values.setdefault("omega", 0.0) + values.setdefault("tmid", 0.0) + return values + + +def planet_dict_transit_errors(planet_dict, limb_darkening=None, fallback=None): + errors = dict(fallback or {}) + planet_dict = planet_dict or {} + + aliases = { + "rprs": ( + "rprsUnc", + "pl_ratrorerr1", + "Rp/Rs Uncertainty", + "Rp/R* Uncertainty", + "Ratio of Planet to Stellar Radius (Rp/Rs) Uncertainty", + "Ratio of Planet to Stellar Radius (Rp/R*) Uncertainty", + ), + "per": ( + "pPerUnc", + "pl_orbpererr1", + "Orbital Period Uncertainty", + ), + "ars": ( + "aRsUnc", + "pl_ratdorerr1", + "a/Rs Uncertainty", + "a/R* Uncertainty", + "Ratio of Distance to Stellar Radius (a/Rs) Uncertainty", + "Ratio of Distance to Stellar Radius (a/R*) Uncertainty", + ), + "inc": ( + "incUnc", + "pl_orbinclerr1", + "Orbital Inclination Uncertainty", + "Orbital Inclination (deg) Uncertainty", + ), + "tmid": ( + "midTUnc", + "pl_tranmiderr1", + "Mid-Transit Time Uncertainty", + ), + } + for target, names in aliases.items(): + for name in names: + if name not in planet_dict: + continue + value = abs(finite_float(planet_dict.get(name))) + if np.isfinite(value): + errors[target] = value + break + + if limb_darkening is not None: + for index, item in enumerate(limb_darkening): + if index > 3: + break + if not isinstance(item, (list, tuple, np.ndarray)) or len(item) < 2: + continue + value = abs(finite_float(item[1])) + if np.isfinite(value): + errors[f"u{index}"] = value + return errors + + +def complete_transit_parameters(parameters): + values = planet_dict_transit_parameters(parameters) + if "per" not in values and "period" in values: + values["per"] = values["period"] + return values + + +def complete_transit_errors(errors): + return planet_dict_transit_errors(errors, fallback=errors) + + +def transit_duration_days(parameters): + values = complete_transit_parameters(parameters) + period = finite_float(values.get("per")) + rprs = finite_float(values.get("rprs")) + ars = finite_float(values.get("ars")) + inc = finite_float(values.get("inc")) + ecc = finite_float(values.get("ecc"), 0.0) + omega = math.radians(finite_float(values.get("omega"), 0.0)) + + if ( + not np.isfinite(period) or period <= 0 + or not np.isfinite(rprs) or rprs < 0 + or not np.isfinite(ars) or ars <= 0 + or not np.isfinite(inc) + or not np.isfinite(ecc) or ecc < 0 or ecc >= 1 + ): + return np.nan + + sin_inc = math.sin(math.radians(inc)) + if not np.isfinite(sin_inc) or sin_inc <= 0: + return np.nan + + denominator = 1.0 + ecc * math.sin(omega) + if not np.isfinite(denominator) or math.isclose(denominator, 0.0): + return np.nan + + impact_scale = ars * (1.0 - ecc ** 2) / denominator + impact_parameter = impact_scale * math.cos(math.radians(inc)) + chord_sq = (1.0 + rprs) ** 2 - impact_parameter ** 2 + if not np.isfinite(chord_sq) or chord_sq <= 0 or impact_scale <= 0: + return np.nan + + argument = math.sqrt(chord_sq) / (ars * sin_inc) + argument = float(np.clip(argument, -1.0, 1.0)) + eccentric_speed_factor = math.sqrt(1.0 - ecc ** 2) / denominator + duration = (period / math.pi) * math.asin(argument) * eccentric_speed_factor + return float(duration) if np.isfinite(duration) and duration > 0 else np.nan + + +def impact_parameter(parameters): + values = complete_transit_parameters(parameters) + ars = finite_float(values.get("ars")) + inc = finite_float(values.get("inc")) + ecc = finite_float(values.get("ecc"), 0.0) + omega = math.radians(finite_float(values.get("omega"), 0.0)) + if not np.isfinite(ars) or not np.isfinite(inc) or not np.isfinite(ecc): + return np.nan + denominator = 1.0 + ecc * math.sin(omega) + if not np.isfinite(denominator) or math.isclose(denominator, 0.0): + return np.nan + return float(ars * (1.0 - ecc ** 2) * math.cos(math.radians(inc)) / denominator) + + +def geometric_observable_depth_fraction(parameters): + values = complete_transit_parameters(parameters) + rprs = finite_float(values.get("rprs")) + b = abs(impact_parameter(values)) + if not np.isfinite(rprs) or rprs < 0 or not np.isfinite(b): + return np.nan + if b >= 1.0 + rprs: + return 0.0 + if b <= abs(1.0 - rprs): + return float(min(rprs ** 2, 1.0)) + if b <= 0: + return float(min(rprs ** 2, 1.0)) + + star_radius = 1.0 + planet_radius = rprs + cos_star = np.clip( + (b ** 2 + star_radius ** 2 - planet_radius ** 2) / (2.0 * b * star_radius), + -1.0, + 1.0, + ) + cos_planet = np.clip( + (b ** 2 + planet_radius ** 2 - star_radius ** 2) / (2.0 * b * planet_radius), + -1.0, + 1.0, + ) + overlap = ( + star_radius ** 2 * math.acos(cos_star) + + planet_radius ** 2 * math.acos(cos_planet) + - 0.5 * math.sqrt( + max( + 0.0, + (-b + star_radius + planet_radius) + * (b + star_radius - planet_radius) + * (b - star_radius + planet_radius) + * (b + star_radius + planet_radius), + ) + ) + ) + return float(np.clip(overlap / math.pi, 0.0, 1.0)) + + +def transit_depth_evaluation_times(parameters, sample_count=2000): + values = complete_transit_parameters(parameters) + tmid = finite_float(values.get("tmid"), 0.0) + period = finite_float(values.get("per")) + duration = transit_duration_days(values) + if np.isfinite(duration) and duration > 0: + half_window = duration + elif np.isfinite(period) and period > 0: + half_window = min(0.2, 0.1 * period) + else: + half_window = 0.2 + half_window = max(float(half_window), 1.0e-4) + return np.linspace(tmid - half_window, tmid + half_window, int(sample_count)) + + +def _load_transit_model(): + try: + from .api.elca import transit + except Exception: + try: + from api.elca import transit + except Exception: + return None + return transit + + +def _depth_fraction_from_model_flux(model_flux): + try: + flux = np.asarray(model_flux, dtype=float).reshape(-1) + except (TypeError, ValueError): + return np.nan + finite = flux[np.isfinite(flux)] + if finite.size == 0: + return np.nan + return float(max(0.0, 1.0 - np.nanmin(finite))) + + +def observable_depth_fraction(parameters, model_flux=None, times=None): + values = complete_transit_parameters(parameters) + if not all(np.isfinite(finite_float(values.get(key))) for key in _REQUIRED_TRANSIT_KEYS): + fallback_depth = _depth_fraction_from_model_flux(model_flux) + return fallback_depth if np.isfinite(fallback_depth) else geometric_observable_depth_fraction(values) + + transit_model = _load_transit_model() + if transit_model is not None: + try: + if times is None: + times = transit_depth_evaluation_times(values) + flux = transit_model(np.asarray(times, dtype=float), values) + depth = _depth_fraction_from_model_flux(flux) + if np.isfinite(depth): + return depth + except Exception: + pass + + fallback_depth = _depth_fraction_from_model_flux(model_flux) + if np.isfinite(fallback_depth): + return fallback_depth + return geometric_observable_depth_fraction(values) + + +def _perturbed_value(key, value): + value = finite_float(value) + if not np.isfinite(value): + return np.nan + if key in ("rprs", "ars", "per"): + return max(value, np.finfo(float).eps) + if key == "ecc": + return float(np.clip(value, 0.0, 0.999999)) + if key == "inc": + return float(np.clip(value, 0.0, 180.0)) + return value + + +def observable_depth_uncertainty_fraction(parameters, errors): + values = complete_transit_parameters(parameters) + errors = complete_transit_errors(errors or {}) + contributions = [] + for key in _DEPTH_ERROR_KEYS: + center = finite_float(values.get(key)) + error = abs(finite_float(errors.get(key))) + if not np.isfinite(center) or not np.isfinite(error) or error <= 0: + continue + + lower_values = dict(values) + upper_values = dict(values) + lower_values[key] = _perturbed_value(key, center - error) + upper_values[key] = _perturbed_value(key, center + error) + lower_depth = observable_depth_fraction(lower_values) + upper_depth = observable_depth_fraction(upper_values) + if np.isfinite(lower_depth) and np.isfinite(upper_depth): + contributions.append(0.5 * abs(upper_depth - lower_depth)) + + if not contributions: + return np.nan + return float(np.sqrt(np.sum(np.square(contributions)))) + + +def observable_depth_percent(parameters, errors=None, model_flux=None, times=None): + depth_fraction = observable_depth_fraction(parameters, model_flux=model_flux, times=times) + if not np.isfinite(depth_fraction): + return np.nan, np.nan + error_fraction = observable_depth_uncertainty_fraction(parameters, errors or {}) + return ( + float(100.0 * depth_fraction), + float(100.0 * error_fraction) if np.isfinite(error_fraction) else np.nan, + ) + + +def fit_transit_depth_summary(fit, prior_parameters=None, prior_errors=None): + parameters = dict(getattr(fit, "parameters", {}) or {}) + errors = dict(getattr(fit, "errors", {}) or {}) + model_flux = getattr(fit, "transit_upsample", None) + times = getattr(fit, "time_upsample", None) + + area_depth, area_error = radius_ratio_area_depth_percent( + parameters.get("rprs"), + errors.get("rprs"), + ) + observable_depth, observable_error = observable_depth_percent( + parameters, + errors, + model_flux=model_flux, + times=times, + ) + + if prior_parameters is None: + prior_parameters = getattr(fit, "prior", None) + prior_depth = np.nan + prior_error = np.nan + if prior_parameters: + prior_depth, prior_error = observable_depth_percent(prior_parameters, prior_errors or {}) + if not np.isfinite(prior_depth): + prior_values = complete_transit_parameters(prior_parameters) + prior_errors = complete_transit_errors(prior_errors or {}) + prior_depth, prior_error = radius_ratio_area_depth_percent( + prior_values.get("rprs"), + prior_errors.get("rprs"), + ) + + delta = np.nan + delta_error = np.nan + if np.isfinite(observable_depth) and np.isfinite(prior_depth): + delta = float(observable_depth - prior_depth) + if np.isfinite(observable_error) and np.isfinite(prior_error): + delta_error = float(np.hypot(observable_error, prior_error)) + + return { + "area_depth": area_depth, + "area_depth_error": area_error, + "observable_depth": observable_depth, + "observable_depth_error": observable_error, + "prior_observable_depth": prior_depth, + "prior_observable_depth_error": prior_error, + "observable_depth_prior_delta": delta, + "observable_depth_prior_delta_error": delta_error, + } diff --git a/exotic/utils.py b/exotic/utils.py index cbc7df76..ffad4da5 100644 --- a/exotic/utils.py +++ b/exotic/utils.py @@ -1,4 +1,6 @@ import logging +from math import isfinite +from pathlib import Path import re import requests from numpy import floor, log10 @@ -13,6 +15,186 @@ log = logging.getLogger(__name__) +_WINDOWS_RESERVED_FILENAME_STEMS = { + 'CON', + 'PRN', + 'AUX', + 'NUL', + *(f'COM{i}' for i in range(1, 10)), + *(f'LPT{i}' for i in range(1, 10)), +} +_WINDOWS_ILLEGAL_FILENAME_CHARS_RE = re.compile(r'[<>:"/\\|?*\x00-\x1f\x7f]') +_FILENAME_WHITESPACE_RE = re.compile(r'\s+') +_COMPACT_EXOPLANET_SUFFIX_RE = re.compile(r'(?<=[0-9A-Z])([b-z])$') +MAX_APPARENT_MAGNITUDE = 30.0 +MAGNITUDE_DECIMAL_PLACES = 4 +MINIMUM_MAGNITUDE_ERROR = 0.001 +AAVSO_OUTPUT_FOLDER_NAME = 'AAVSO_Files' +BOOLEAN_CONFIG_TRUE_STRINGS = frozenset(('y', 'yes', 'true', '1', 'on')) +BOOLEAN_CONFIG_FALSE_STRINGS = frozenset(('n', 'no', 'false', '0', 'off', '')) + + +def coerce_boolean_config_value(value): + """Return a configured boolean, or ``None`` when the value is not boolean-like. + + JSON booleans and numeric 1/0 are accepted directly. String values are + case-insensitive and accept y/n, yes/no, true/false, 1/0, and on/off. + """ + + if isinstance(value, bool): + return value + if isinstance(value, (int, float)): + if not isfinite(value): + return None + if value == 1: + return True + if value == 0: + return False + return None + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in BOOLEAN_CONFIG_TRUE_STRINGS: + return True + if normalized in BOOLEAN_CONFIG_FALSE_STRINGS: + return False + return None + + +def aavso_output_directory(root): + """Return the dedicated AAVSO output directory, creating it when needed.""" + + output_directory = Path(root) / AAVSO_OUTPUT_FOLDER_NAME + output_directory.mkdir(parents=True, exist_ok=True) + return output_directory + + +def format_aavso_exoplanet_name(value): + """Separate a compact trailing planet letter for the AAVSO header.""" + + name = str(value or '').strip() + return _COMPACT_EXOPLANET_SUFFIX_RE.sub(r' \1', name) + + +def _clean_filename_text(value): + cleaned = _WINDOWS_ILLEGAL_FILENAME_CHARS_RE.sub('-', str(value or '')) + cleaned = _FILENAME_WHITESPACE_RE.sub('', cleaned) + return cleaned.rstrip(' .') + + +def sanitize_filename_component(value, fallback='output'): + """Return one filename component that is safe on Windows, macOS, and Linux.""" + + cleaned = _clean_filename_text(value) + if cleaned in {'', '.', '..'}: + cleaned = _clean_filename_text(fallback) + if cleaned in {'', '.', '..'}: + cleaned = 'output' + device_stem = cleaned.split('.', 1)[0].upper() + if device_stem in _WINDOWS_RESERVED_FILENAME_STEMS: + cleaned = f'_{cleaned}' + return cleaned + + +def filename_date_token(value): + """Return YYYY-MM-DD when a filename date includes a time component.""" + + text = str(value or '').strip() + match = re.match(r'(\d{4})[-/]?(\d{2})[-/]?(\d{2})', text) + if match: + return f'{match.group(1)}-{match.group(2)}-{match.group(3)}' + return text + + +def safe_output_filename(prefix, *parts, extension): + """Build a filename from EXOTIC output labels without illegal path characters.""" + + stem_parts = [str(prefix), *(str(part) for part in parts)] + safe_stem = sanitize_filename_component('_'.join(stem_parts), fallback=str(prefix or 'output')) + ext = _FILENAME_WHITESPACE_RE.sub('', str(extension or '')) + if ext and not ext.startswith('.'): + ext = f'.{ext}' + return f'{safe_stem}{ext}' + + +def parse_finite_float(value, default=None): + try: + parsed = float(value) + except (TypeError, ValueError): + return default + return parsed if isfinite(parsed) else default + + +def is_usable_apparent_magnitude(value, max_magnitude=MAX_APPARENT_MAGNITUDE): + parsed = parse_finite_float(value) + return parsed is not None and parsed <= max_magnitude + + +def format_magnitude(value, default="na", digits=MAGNITUDE_DECIMAL_PLACES, + max_magnitude=MAX_APPARENT_MAGNITUDE): + parsed = parse_finite_float(value) + if parsed is None or parsed > max_magnitude: + return default + return f"{parsed:.{digits}f}" + + +def rounded_magnitude_value(value, default=None, digits=MAGNITUDE_DECIMAL_PLACES, + max_magnitude=MAX_APPARENT_MAGNITUDE): + parsed = parse_finite_float(value) + if parsed is None or parsed > max_magnitude: + return default + return round(parsed, digits) + + +def normalized_magnitude_error(value, default=None, minimum=MINIMUM_MAGNITUDE_ERROR, + max_magnitude=MAX_APPARENT_MAGNITUDE): + parsed = parse_finite_float(value) + if parsed is None: + return default + parsed = abs(parsed) + if parsed > max_magnitude: + return default + return max(parsed, minimum) + + +def format_magnitude_error(value, default="na", digits=MAGNITUDE_DECIMAL_PLACES, + minimum=MINIMUM_MAGNITUDE_ERROR, + max_magnitude=MAX_APPARENT_MAGNITUDE): + parsed = normalized_magnitude_error( + value, + default=None, + minimum=minimum, + max_magnitude=max_magnitude, + ) + if parsed is None: + return default + return f"{parsed:.{digits}f}" + + +def rounded_magnitude_error(value, default=None, digits=MAGNITUDE_DECIMAL_PLACES, + minimum=MINIMUM_MAGNITUDE_ERROR, + max_magnitude=MAX_APPARENT_MAGNITUDE): + parsed = normalized_magnitude_error( + value, + default=None, + minimum=minimum, + max_magnitude=max_magnitude, + ) + if parsed is None: + return default + return round(parsed, digits) + + +def magnitude_text(band, magnitude, magnitude_error=None): + formatted_mag = format_magnitude(magnitude, default=None) + if formatted_mag is None: + return None + + formatted_error = format_magnitude_error(magnitude_error, default=None) + if formatted_error is None: + return f"{band}={formatted_mag}" + return f"{band}={formatted_mag} +/- {formatted_error}" + + def user_input(prompt, type_, values=None, max_tries=1000): """ Captures user_input and casts it to the expected type @@ -190,6 +372,68 @@ def round_to_2(*args): return round(x, roundval) +def _two_significant_figure_decimal_places(uncertainty): + """Return the decimal place needed to show an uncertainty with two sig figs.""" + + uncertainty = float(uncertainty) + if not isfinite(uncertainty) or uncertainty < 0: + raise ValueError("uncertainty must be a finite, non-negative number") + if uncertainty == 0: + return 2 + + exponent = int(floor(log10(abs(uncertainty)))) + decimal_places = 1 - exponent + + # A carry can change the exponent (for example, 0.00999 -> 0.010). + rounded_uncertainty = round(uncertainty, decimal_places) + if rounded_uncertainty: + rounded_exponent = int(floor(log10(abs(rounded_uncertainty)))) + decimal_places = 1 - rounded_exponent + return decimal_places + + +def _format_at_decimal_place(value, decimal_places): + """Format a number at a decimal place, including insignificant zeroes.""" + + value = float(value) + if not isfinite(value): + raise ValueError("value must be a finite number") + if decimal_places >= 0: + return f"{value:.{decimal_places}f}" + return f"{round(value, decimal_places):.0f}" + + +def format_value_and_uncertainty(value, uncertainty): + """Return value/error text with a two-significant-figure uncertainty. + + Both strings end at the same decimal place. Unlike ``round_to_2``, this is + a reporting helper: it deliberately retains trailing zeroes which carry + precision information. + """ + + decimal_places = _two_significant_figure_decimal_places(uncertainty) + return ( + _format_at_decimal_place(value, decimal_places), + format_uncertainty(uncertainty), + ) + + +def format_uncertainty(uncertainty): + """Format an uncertainty with exactly two significant figures.""" + + decimal_places = _two_significant_figure_decimal_places(uncertainty) + if decimal_places < 0: + return f"{float(uncertainty):.1e}" + return _format_at_decimal_place(uncertainty, decimal_places) + + +def format_value_with_uncertainty(value, uncertainty): + """Return ``value +/- uncertainty`` using matched two-sig-fig precision.""" + + value_text, uncertainty_text = format_value_and_uncertainty(value, uncertainty) + return f"{value_text} +/- {uncertainty_text}" + + # Credit: Kalee Tock def get_val(hdr, ks): """ @@ -269,8 +513,9 @@ def process_lat_long(val, key): Parameters ---------- val : str - either a longitude or latitude coordinate, with a preceding + or -, - expressed in _either_ HH:MM:SS or degree values. ex: +152.51 or +37:2:24. + Either a longitude or latitude coordinate expressed in HH:MM:SS or + decimal degrees. It may use a leading + or - or a FITS-style N/S/E/W + hemisphere letter. Examples: +152.51, +37:2:24, or 16 30 39.7 W. key : str expects "longitude" or "latitude" @@ -280,26 +525,48 @@ def process_lat_long(val, key): longitude or latitude expressed in degree coordinates with a preceding + or -. Six digits of precision after the decimal. ex: +152.510000 """ - m = re.search(r"\'?([+-]?\d+)[\s:](\d+)[\s:](\d+\.?\d*)", val) or \ - re.search(r"\'?([+-]?\d+)[\s:](\d+\.\d*)", val) - if m: - try: - deg, min, sec = float(m.group(1)), float(m.group(2)), float(m.group(3)) - except IndexError: - deg, min, sec = float(m.group(1)), float(m.group(2)), 0 - if deg < 0: - v = deg - (((60 * min) + sec) / 3600) - else: - v = deg + (((60 * min) + sec) / 3600) - return add_sign(v) - - m = re.search("^\'?([+-]?\d+\.\d+)", val) + text = str(val).strip() + coordinate_type = str(key).strip().lower() + valid_hemispheres = { + "latitude": {"N", "S"}, + "longitude": {"E", "W"}, + }.get(coordinate_type) + hemisphere = None + + # FITS writers commonly append a hemisphere letter to an otherwise + # unsigned decimal or sexagesimal coordinate. A hemisphere overrides a + # redundant leading sign so that ``-16 30 W`` is not double-negated. + trailing_hemisphere = re.search(r"([NSEW])\s*$", text) + leading_hemisphere = re.match(r"\s*([NSEW])(?=\s|[+-]?\d)", text) + hemisphere_match = trailing_hemisphere or leading_hemisphere + if hemisphere_match: + hemisphere = hemisphere_match.group(1) + if valid_hemispheres is not None and hemisphere not in valid_hemispheres: + print(f"Cannot match value {val}, which is meant to be {key}.") + return None + start, end = hemisphere_match.span(1) + text = f"{text[:start]}{text[end:]}".strip() - if m: - v = float(m.group(1)) - return add_sign(v) - else: + number_tokens = re.findall(r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)", text) + if not 1 <= len(number_tokens) <= 3: + print(f"Cannot match value {val}, which is meant to be {key}.") + return None + if len(number_tokens) == 1 and hemisphere is None \ + and "." not in number_tokens[0] and number_tokens[0][0] not in "+-": + # Preserve the historical rejection of an unsigned integer while + # accepting one when a hemisphere supplies the otherwise missing sign. print(f"Cannot match value {val}, which is meant to be {key}.") + return None + + degrees = float(number_tokens[0]) + minutes = abs(float(number_tokens[1])) if len(number_tokens) >= 2 else 0.0 + seconds = abs(float(number_tokens[2])) if len(number_tokens) >= 3 else 0.0 + magnitude = abs(degrees) + minutes / 60.0 + seconds / 3600.0 + if hemisphere: + sign = -1.0 if hemisphere in {"S", "W"} else 1.0 + else: + sign = -1.0 if number_tokens[0].startswith("-") else 1.0 + return add_sign(sign * magnitude) # Credit: Kalee Tock diff --git a/exotic/version.py b/exotic/version.py index 376d9ccc..26a6c390 100644 --- a/exotic/version.py +++ b/exotic/version.py @@ -1 +1 @@ -__version__ = '4.3.1' +__version__ = '4.4.0' diff --git a/inits.json b/inits.json index 49d057f9..47307c64 100644 --- a/inits.json +++ b/inits.json @@ -1,97 +1,180 @@ { "inits_guide": { - "Title": "EXOTIC's Initialization File", - "Comment": "Please answer all the following requirements below by following the format of the given", - "Comment1": "sample dataset HAT-P-32 b. Edit this file as needed to match the data wanting to be reduced.", - "Comment2": "Do not delete areas where there are quotation marks, commas, and brackets.", - "Comment3": "The inits_guide dictionary (these lines of text) does not have to be edited", - "Comment4": "and is only here to serve as a guide. Will be updated per user's advice.", - "Image Calibrations Directory Guide": "Enter in the path to image calibrations or enter in null for none.", - "Planetary Parameters Guide": "For planetary parameters that are not filled in, enter in null.", - "Comparison Star(s) Guide": "Up to 10 comparison stars can be added following the format given below.", - "Obs. Latitude Guide": "Indicate the sign (+ North, - South) before the degrees. Needs to be in decimal or HH:MM:SS format.", - "Obs. Longitude Guide": "Indicate the sign (+ East, - West) before the degrees. Needs to be in decimal or HH:MM:SS format.", - "Camera Type (1)": "If you are using a CMOS, please enter CCD in 'Camera Type (CCD or DSLR)' and then note", - "Camera Type (2)": "your actual camera type under 'Observing Notes'.", - "Plate Solution": "For your image to be given a plate solution, type y.", - "Plate Solution Disclaimer": "One of your imaging files will be publicly viewable on nova.astrometry.net.", - "Standard Filter": "To use EXOTIC standard filters, type only the filter name.", - "Custom Filter": "To use a custom filter, enter in the FWHM in optional_info.", - "Target Star RA": "Must be in HH:MM:SS sexagesimal format.", - "Target Star DEC": "Must be in +/-DD:MM:SS sexagesimal format with correct sign at the beginning (+ or -).", - "Demosaic Format": "Optional control for handling Bayer pattern color images - to use, provide Bayer color patttern of your camera (RGGB, BGGR, GRBG, GBRG) - null (no color processing) is default", - "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", - "Formatting of null": "Due to the file being a .json, null is case sensitive and must be spelled as shown.", - "Decimal Format": "Leading zero must be included when appropriate (Ex: 0.32, .32 or 00.32 causes errors.)." + "Title": "EXOTIC's Initialization File", + "Comment": "Please answer all the following requirements below by following the format of the given", + "Comment1": "sample dataset HAT-P-32 b. Edit this file as needed to match the data wanting to be reduced.", + "Comment2": "Do not delete areas where there are quotation marks, commas, and brackets.", + "Comment3": "The inits_guide dictionary (these lines of text) does not have to be edited", + "Comment4": "and is only here to serve as a guide. Will be updated per user's advice.", + "Image Calibrations Directory Guide": "Enter in the path to image calibrations or enter in null for none.", + "Planetary Parameters Guide": "For planetary parameters that are not filled in, enter in null.", + "Comparison Star(s) Guide": "Provide comparison stars either as X/Y pixels or as RA/Dec coordinates, but not both. RA/Dec accepts decimal degrees or sexagesimal strings and requires a usable reference-image WCS.", + "Obs. Latitude Guide": "Indicate the sign (+ North, - South) before the degrees. Needs to be in decimal or HH:MM:SS format.", + "Obs. Longitude Guide": "Indicate the sign (+ East, - West) before the degrees. Needs to be in decimal or HH:MM:SS format.", + "Camera Type (1)": "If you are using a CMOS, please enter CCD in 'Camera Type (CCD or DSLR)' and then note", + "Camera Type (2)": "your actual camera type under 'Observing Notes'.", + "Observatory Full Title": "Optional full observatory name. If present, EXOTIC writes it to AAVSO output as OBSNAME.", + "Plate Solution": "For your image to be given a plate solution, type y.", + "Plate Solution Disclaimer": "One of your imaging files will be publicly viewable on nova.astrometry.net.", + "Standard Filter": "To use EXOTIC standard filters, type only the filter name.", + "Custom Filter": "To use a custom filter, enter in the FWHM in optional_info.", + "Target Star RA": "Must be in HH:MM:SS sexagesimal format.", + "Target Star DEC": "Must be in +/-DD:MM:SS sexagesimal format with correct sign at the beginning (+ or -).", + "Demosaic Format": "Optional control for handling Bayer pattern color images - to use, provide Bayer color patttern of your camera (RGGB, BGGR, GRBG, GBRG) - null (no color processing) is default", + "Demosaic Output": "Select how to process color data (gray for grayscale, red or green or blue for single color channel, blueblock for grayscale without blue, [ R, G, B ] for custom weights for mixing colors. green is default", + "Fast Aperture Mask": "Default false/exact mode for fractional-pixel aperture photometry. Set optional_info 'Fast Aperture Mask (y/n)' to true to opt into center-based masks for speed.", + "Ignore Header WCS": "Set optional_info 'Ignore WCS in Header and Do Manual Alignment? (y/n)' to y to ignore FITS header WCS and force legacy image-to-image alignment. Default n.", + "Pixel Alignment Fallback": "Set optional_info 'allow_pixel_alignment_fallback' to false to require WCS-only processing and drop every frame without celestial WCS. Default true; EXOTIC prefers WCS when coverage is consistent and otherwise retains the sequence for legacy alignment.", + "Bad WCS Threshold Percent": "When allow_pixel_alignment_fallback is true, set optional_info 'bad_wcs_threshold_percent' to the maximum percent of images allowed to lack celestial WCS while still using WCS-only processing. Below the threshold, missing-WCS frames are dropped; at or above it, all frames are retained for alignment fallback. Default 3.", + "Pointing Rejection Sigma": "Set optional_info 'pointing_rejection_sigma' to a positive sigma threshold to reject frames whose WCS-derived or alignment-derived pointings are strong outliers from the dataset median pointing before photometry. Leave blank/null or set to 0/off to disable. Default disabled.", + "Prefer Pixel Coordinates Over WCS": "Set optional_info 'prefer_pixel_values_over_wcs_for_target' to y to keep the entered target pixel coordinates when they conflict with WCS-derived target coordinates. Default n.", + "Vertical Flux Normalization": "Set optional_info 'disable vertical flux normalization' to true to disable the default a0 baseline bound of [0.95, 1.05]. Default false.", + "Stellar Variability Only": "Set optional_info 'stellar_variability_only' to true to skip transit fitting, use the default calibrated comparison-star ensemble, and discard predicted ingress-to-egress transit-window points. Default false.", + "Stellar Variability Ensemble": "Set optional_info 'use_ensemble_photometry_for_stellar_variability' to false to disable the default calibrated ensemble in stellar_variability_only runs and restore single-comparison selection by out-of-transit scatter. The default ensemble automatically finds bright catalog-calibrated field-star candidates, removes saturated and VSX-variable stars, sigma-clips high catalog magnitude uncertainties, and retains up to maximum_number_of_ensemble_comparisons_for_stellar_variability stars closest to the target in catalog colour and magnitude. EnsembleSelection JSON records the target and comparison colours, magnitudes, errors, and selection deltas beside the final results.", + "Apparent Magnitudes Required": "Set optional_info 'require_apparent_magnitudes' to false when catalogue-calibrated apparent magnitudes are not required. EXOTIC still attempts apparent outputs when calibration is available, but differential-magnitude CSV and plot products never depend on a catalogue magnitude. Stellar-variability magnitudes use the raw target/reference ratio with no airmass correction. Default true.", + "Use Exactly Supplied Comparisons": "Set optional_info 'use_exactly_the_comps_provided' to true to use only the comparison coordinates supplied in user_info, whether supplied as X/Y pixels or RA/Dec, without automatic replacement, VSX rejection, stability vetting, ranking, or ensemble-size limiting. One supplied comparison is used alone; two or more are all used together as one fixed ensemble. Default false.", + "Maximum Transit Ensemble Comparisons": "Set optional_info 'maximum_number_of_ensemble_comparisons_for_transit' to the largest number of comparison stars used by the transit-fit ensemble. Default 5; values must be integers of at least 2, with no configured upper limit.", + "Maximum Stellar-Variability Ensemble Comparisons": "Set optional_info 'maximum_number_of_ensemble_comparisons_for_stellar_variability' to the largest number of comparison stars used by stellar-variability-only and fortuitous-variable ensembles. Default 5; values must be integers of at least 2, with no configured upper limit. Large variability ensembles increase photometry work and can reject more frames because every selected ensemble member must be usable in a retained frame.", + "Fortuitous Variable Photometry": "Set optional_info 'photometer_fortuitous_variables' to false to disable the default full-field VSX search and independent calibrated photometry of retained variables. Stars are retained only when their reference-image count-rate estimate has an internal error below 0.05 mag. Each VSX target uses its own frame-level saturation mask; exoplanet-target saturation does not reject that image from the VSX run. Outputs are written under variables/optimal_variables/ for VSX period <= 10 days and amplitude >= 0.3 mag, otherwise under variables/normal/. Skipped variables are recorded only in the shared variables manifest and do not receive an object directory.", + "Fortuitous Variable Single Comparison": "Set optional_info 'use_single_comparison_for_fortuitous_variables' to false to use the calibrated comparison-star ensemble for fortuitous VSX targets. The default true selects one unsaturated, non-variable, catalog-calibrated comparison star closest to each VSX target in catalog colour and magnitude.", + "Automatic AAVSO V Calibration Fallback": "For V-family observations, including Clear, CV, and bv, EXOTIC always requires V-band comparison magnitudes. If the NextAstro photometry server supplies no usable V calibration, EXOTIC automatically queries AAVSO VSP and adds matched or discovered V-sequence stars to the same single-comparison or ensemble calibration pool, even when 'Add Comparison Stars from AAVSO?' is n.", + "NextAstro VSX Cache First": "Set optional_info 'use_nextastro_vsx_cache_first' to true to query the NextAstro /vsx_query field cache before AAVSO VSX. The default is false. Empty or failed cache lookups fall back to AAVSO; legacy cache responses lacking the full period/amplitude schema are enriched from AAVSO.", + "Detect Bad Pixels Before Photometry": "Set optional_info 'detect_bad_pixels_before_photometry' to y to scan the frame stack for persistent isolated high-count bad pixels before plate-solve checks and photometry, save the detection count image and mask into working_artifacts/, and median-8 repair those pixels before centroiding and photometry. Default n.", + "Multiprocess Bad-Pixel Precheck": "Set optional_info 'multiprocess_bad_pixel_precheck' to y or a positive process count to scan bad pixels in parallel. Default n.", + "Out-of-Transit Baseline Detrending": "Set optional_info 'detrend_on_outoftransit_baseline' to true to run a second-pass final fit after dividing out a weighted linear trend fit only to the modeled out-of-transit baseline before ingress and after egress. Default true.", + "Final Fit Baseline Duration Multiplier": "Set optional_info 'final_fit_baseline_duration_multiplier' to the number of fitted transit durations to keep as baseline before ingress and after egress during the automatic final-fit prefit/refit. Default 1.0.", + "EEBLS Tmid Initializer": "Set optional_info 'use_eebls_to_initialize_tmid_and_bounds' to y to run a fixed-period box least squares search over the light curve, use the strongest bracketed transit-like signal to initialize Tmid, and narrow the Tmid search range before fitting. Default y.", + "Pick Comparison by EEBLS SNR": "Set optional_info 'pick_comparison_by_eebls_snr' to y to use EEBLS depth SNR as an earlier tie-break when KTMF scores do not settle the comparison-star choice. Default y.", + "Expected-Value Transit QC": "Set optional_info 'use_deviation_from_expected_transit_in_qc' to true to reject transit fits whose fitted Rp/R* strays too far from the published expected value using the fitted Rp/R* uncertainty only. Default true.", + "Expected-Value Transit QC Sigma": "Set optional_info 'deviation_from_expected_transit_in_qc_sigma' to the sigma threshold used by the expected-value transit QC rejection. Default 5.", + "Impact Parameter Fit": "Set optional_info 'use_impactparameter_rather_than_inclination_to_fit' to y to sample impact parameter instead of inclination in nested fitting and triangle plots. Default y.", + "UltraNest Live Points": "Set optional_info 'minimum number of live points for ultranest' to a positive integer to control UltraNest's min_num_live_points. Default 200.", + "Fast UltraNest Before Final Run": "Set optional_info 'run fast ultranest before final run' to y to run comparison-candidate UltraNest searches on a binned light curve of at most 20 points when more than 60 points are available, then rerun the selected final fit on the full light curve. Default y.", + "Sparse Posterior Live-Point Retry": "Set optional_info 'use_sparse_posterior_live_point_retry' to y to rank comparison-star candidates at the configured UltraNest live-point count, then run or continue the chosen final comparison-star fit with 5x additional minimum live points. Standalone final fits still only continue when Rp/Rs, Tmid, or a/Rs posteriors are too sparse. Set to n to disable. Default y.", + "Use PSF Photometry": "Set optional_info 'use_psf_photometry' to y to keep PSF photometry in the method search, or n to disable PSF photometry entirely. Default y.", + "Use Aperture Photometry": "Set optional_info 'use_aperture_photometry' to y to keep aperture photometry in the method search, or n to disable aperture photometry entirely. Default y.", + "Adaptive Apertures": "Set optional_info 'use_adaptive_apertures' to true to evaluate aperture candidates in PSF sigma units and rescale the actual aperture/annulus radii frame-by-frame from the measured PSF width. Default false.", + "Aperture Corrections and Full Image FWHM": "Set optional_info 'use_aperture_corrections_and_full_image_fwhm' to true to estimate image FWHM from isolated field stars and apply isolated-star aperture corrections. Default false.", + "Reject Overexposed Stars": "Set optional_info 'reject_overexposed_stars' to true to reject overexposed target frames and overexposed comparison-star measurements. Default true.", + "Saturation Value": "Set optional_info 'saturation_value' to the detector saturation value in the same units as the image pixels. If omitted/default, EXOTIC uses FITS SATURATE when available, maps TELESCOP Cecilia to 4096, otherwise uses 65535.", + "Overexposure Threshold Fraction": "Set optional_info 'overexposure_threshold_fraction' to the fraction of saturation used for rejection. Default 0.9, so pixels above 0.9 * saturation_value are rejected.", + "Skip Low Comparison Coverage Rejection": "Set optional_info 'skip_low_comparison_coverage_rejection' to y to disable EXOTIC's default rejection of comparison stars that are valid in far fewer frames than the rest of the comparison-star field. Default n.", + "Fit Lightcurve to Every Comparison Candidate": "Set optional_info 'fit_lightcurve_to_every_comparison_candidate' to y to save one target lightcurve fit plot per comparison star into working_artifacts/ using the selected photometry setup. Default n.", + "Require Comparison Star": "Set optional_info 'require_comp_star' to y to require an actual comparison star for the best-fit photometry result.", + "Target-Driven Comparison Selection": "Set optional_info 'Use target-driven comp selection rather than comp-driven comp selection' to y to force the legacy target-driven comparison-star selection path. Default n.", + "Boolean Values": "All boolean settings accept JSON true/false, numeric 1/0, or case-insensitive strings y/n. The equivalent strings yes/no and on/off are also accepted.", + "Formatting of null": "Due to the file being a .json, null is case sensitive and must be spelled as shown.", + "Decimal Format": "Leading zero must be included when appropriate (Ex: 0.32, .32 or 00.32 causes errors.)." }, "user_info": { - "Directory with FITS files": "/Users/rzellem/Documents/EXOTIC/sample-data/HatP32Dec202017", - "Directory to Save Plots": "/Users/rzellem/Documents/EXOTIC/sample-data/", - "Directory of Flats": null, - "Directory of Darks": null, - "Directory of Biases": null, - - "AAVSO Observer Code (blank if none)": "RTZ", - "Secondary Observer Codes (blank if none)": "", - - "Observation date": "17-December-2017", - "Obs. Latitude": "+32.41638889", - "Obs. Longitude": "-110.73444444", - "Obs. Elevation (meters)": 2616, - "Camera Type (CCD or DSLR)": "CCD", - "Pixel Binning": "1x1", - "Filter Name (aavso.org/filters)": "CV", - "Observing Notes": "Weather, seeing was nice.", - - "Plate Solution? (y/n)": "y", - "Add Comparison Stars from AAVSO? (y/n)": "y", - - "Target Star X & Y Pixel": "[424, 286]", - "Comparison Star(s) X & Y Pixel": "[[465, 183], [512, 263], [], [], [], [], [], [], [], []]", - - "Demosaic Format": null, - "Demosaic Output": null + "Directory with FITS files": "/Users/rzellem/Documents/EXOTIC/sample-data/HatP32Dec202017", + "Directory to Save Plots": "/Users/rzellem/Documents/EXOTIC/sample-data/", + "Directory of Flats": null, + "Directory of Darks": null, + "Directory of Biases": null, + "AAVSO Observer Code (blank if none)": "RTZ", + "Secondary Observer Codes (blank if none)": "", + "Observatory Full Title": "", + "Observation date": "17-December-2017", + "Obs. Latitude": "+32.41638889", + "Obs. Longitude": "-110.73444444", + "Obs. Elevation (meters)": 2616, + "Camera Type (CCD or DSLR)": "CCD", + "Pixel Binning": "1x1", + "Filter Name (aavso.org/filters)": "CV", + "Observing Notes": "Weather, seeing was nice.", + "Plate Solution? (y/n)": true, + "Add Comparison Stars from AAVSO? (y/n)": false, + "Target Star X & Y Pixel": "[424, 286]", + "Comparison Star(s) X & Y Pixel": "[[465, 183], [512, 263], [], [], [], [], [], [], [], []]", + "Comparison Star(s) RA & Dec": null, + "Demosaic Format": null, + "Demosaic Output": null }, "planetary_parameters": { - "Target Star RA": "02:04:10", - "Target Star Dec": "+46:41:23", - "Planet Name": "HAT-P-32 b", - "Host Star Name": "HAT-P-32", - "Orbital Period (days)": 2.1500082, - "Orbital Period Uncertainty": 1.3e-07, - "Published Mid-Transit Time (BJD-UTC)": 2455867.402743, - "Mid-Transit Time Uncertainty": 4.9e-05, - "Ratio of Planet to Stellar Radius (Rp/Rs)": 0.14886235252742716, - "Ratio of Planet to Stellar Radius (Rp/Rs) Uncertainty": 0.0005539487393037134, - "Ratio of Distance to Stellar Radius (a/Rs)": 5.344, - "Ratio of Distance to Stellar Radius (a/Rs) Uncertainty": 0.039496835316262996, - "Orbital Inclination (deg)": 88.98, - "Orbital Inclination (deg) Uncertainty": 0.7602631123499285, - "Orbital Eccentricity (0 if null)": 0.159, - "Argument of Periastron (deg)": 50, - "Star Effective Temperature (K)": 6001.0, - "Star Effective Temperature (+) Uncertainty": 88.0, - "Star Effective Temperature (-) Uncertainty": -88.0, - "Star Metallicity ([FE/H])": -0.16, - "Star Metallicity (+) Uncertainty": 0.08, - "Star Metallicity (-) Uncertainty": -0.08, - "Star Surface Gravity (log(g))": 4.22, - "Star Surface Gravity (+) Uncertainty": 0.04, - "Star Surface Gravity (-) Uncertainty": -0.04, - "Star Distance (pc)": 289.21, - "Star Proper Motion RA (mas/yr)": -9.82, - "Star Proper Motion DEC (mas/yr)": 3.48 + "Target Star RA": "02:04:10", + "Target Star Dec": "+46:41:23", + "Planet Name": "HAT-P-32 b", + "Host Star Name": "HAT-P-32", + "Orbital Period (days)": 2.1500082, + "Orbital Period Uncertainty": 1.3e-07, + "Published Mid-Transit Time (BJD-UTC)": 2455867.402743, + "Mid-Transit Time Uncertainty": 4.9e-05, + "Ratio of Planet to Stellar Radius (Rp/Rs)": 0.14886235252742716, + "Ratio of Planet to Stellar Radius (Rp/Rs) Uncertainty": 0.0005539487393037134, + "Ratio of Distance to Stellar Radius (a/Rs)": 5.344, + "Ratio of Distance to Stellar Radius (a/Rs) Uncertainty": 0.039496835316262996, + "Orbital Inclination (deg)": 88.98, + "Orbital Inclination (deg) Uncertainty": 0.7602631123499285, + "Orbital Eccentricity (0 if null)": 0.159, + "Argument of Periastron (deg)": 50, + "Star Effective Temperature (K)": 6001.0, + "Star Effective Temperature (+) Uncertainty": 88.0, + "Star Effective Temperature (-) Uncertainty": -88.0, + "Star Metallicity ([FE/H])": -0.16, + "Star Metallicity (+) Uncertainty": 0.08, + "Star Metallicity (-) Uncertainty": -0.08, + "Star Surface Gravity (log(g))": 4.22, + "Star Surface Gravity (+) Uncertainty": 0.04, + "Star Surface Gravity (-) Uncertainty": -0.04, + "Star Distance (pc)": 289.21, + "Star Proper Motion RA (mas/yr)": -9.82, + "Star Proper Motion DEC (mas/yr)": 3.48 }, "optional_info": { - "Pre-reduced File:": "/sample-data/NormalizedFlux_HAT-P-32 b_December 17, 2017.txt", - "Pre-reduced File Time Format (BJD_TDB, JD_UTC, MJD_UTC)": "BJD_TDB", - "Pre-reduced File Units of Flux (flux, magnitude, millimagnitude)": "flux", - - "Filter Minimum Wavelength (nm)": null, - "Filter Maximum Wavelength (nm)": null, - - "Image Scale (Ex: 5.21 arcsecs/pixel)": null, - - "Exposure Time (s)": 60.0 + "Pre-reduced File:": "/sample-data/NormalizedFlux_HAT-P-32 b_December 17, 2017.txt", + "Pre-reduced File Time Format (BJD_TDB, JD_UTC, MJD_UTC)": "BJD_TDB", + "Pre-reduced File Units of Flux (flux, magnitude, millimagnitude)": "flux", + "Filter Minimum Wavelength (nm)": null, + "Filter Maximum Wavelength (nm)": null, + "Calculate Limb Darkening Coefficients with Uncertainties? (y/n)": null, + "Fast Aperture Mask (y/n)": false, + "Ignore WCS in Header and Do Manual Alignment? (y/n)": false, + "allow_pixel_alignment_fallback": true, + "bad_wcs_threshold_percent": 3.0, + "pointing_rejection_sigma": null, + "prefer_pixel_values_over_wcs_for_target": false, + "disable vertical flux normalization": false, + "stellar_variability_only": false, + "use_ensemble_photometry_for_stellar_variability": true, + "require_apparent_magnitudes": true, + "use_exactly_the_comps_provided": false, + "maximum_number_of_ensemble_comparisons_for_transit": 5, + "maximum_number_of_ensemble_comparisons_for_stellar_variability": 5, + "photometer_fortuitous_variables": true, + "use_single_comparison_for_fortuitous_variables": true, + "use_nextastro_vsx_cache_first": false, + "detect_bad_pixels_before_photometry": false, + "multiprocess_bad_pixel_precheck": true, + "detrend_on_outoftransit_baseline": true, + "final_fit_baseline_duration_multiplier": 1.0, + "use_eebls_to_initialize_tmid_and_bounds": true, + "pick_comparison_by_eebls_snr": true, + "use_deviation_from_expected_transit_in_qc": true, + "deviation_from_expected_transit_in_qc_sigma": 5.0, + "use_impactparameter_rather_than_inclination_to_fit": true, + "minimum number of live points for ultranest": 200, + "run fast ultranest before final run": true, + "use_sparse_posterior_live_point_retry": true, + "use_psf_photometry": true, + "use_aperture_photometry": true, + "use_adaptive_apertures": false, + "use_aperture_corrections_and_full_image_fwhm": false, + "reject_overexposed_stars": true, + "saturation_value": 65535, + "overexposure_threshold_fraction": 0.9, + "gain_electrons_per_adu": null, + "read_noise_electrons": null, + "dark_current_electrons_per_second_per_pixel": null, + "flat_field_fractional_error": null, + "telescope_aperture_m": null, + "scintillation_coefficient": null, + "skip_low_comparison_coverage_rejection": false, + "fit_lightcurve_to_every_comparison_candidate": false, + "Use target-driven comp selection rather than comp-driven comp selection": false, + "require_comp_star": true, + "Image Scale (Ex: 5.21 arcsecs/pixel)": null, + "Pixel Scale (arsec/pixel)": null, + "Exposure Time (s)": 60.0 } -} \ No newline at end of file +} diff --git a/output/pdf/EXOTIC_inits_default_options.pdf b/output/pdf/EXOTIC_inits_default_options.pdf new file mode 100644 index 00000000..4e025a7d Binary files /dev/null and b/output/pdf/EXOTIC_inits_default_options.pdf differ diff --git a/requirements.txt b/requirements.txt index 6b8f125f..3f4b7730 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,9 @@ astroalign~=2.6.0 astropy~=6.1 astroquery~=0.4.7 barycorrpy~=0.4.4 -colour_demosaicing~=0.2.6 -dynesty~=1.2.3;platform_system=='Windows' +bottleneck~=1.4.2 +colour_demosaicing==0.2.6 +colour-science>=0.4.4,<0.4.7 holoviews~=1.19.1 importlib-metadata>=3.6;python_version<='3.7' imreg_dft~=2.0.0 @@ -11,9 +12,10 @@ LDTk~=1.8.4 lmfit~=1.3.2 matplotlib~=3.9.2 numba~=0.59.1 -numpy~=1.26.4 -pandas~=2.2.3 +numpy==1.26.4 +pandas>=2.2.2,<2.3 panel~=1.5.2 +bokeh>=3.5,<3.7 photutils~=2.0.0 pylightcurve>=4.0.1,<5 python_dateutil~=2.9 @@ -23,6 +25,9 @@ rebound~=4.4.3 requests~=2.32.3 scipy~=1.14.1 scikit-image~=0.24.0 +tifffile<2026.4.11 statsmodels~=0.14.4 tenacity~=9.0 -ultranest~=3.6.5;platform_system!='Windows' +mpi4py>=4.0;platform_system=='Linux' +ultranest==4.5.0 +zstandard~=0.23.0 diff --git a/scripts/expanded_prior_real_data_trial.py b/scripts/expanded_prior_real_data_trial.py new file mode 100644 index 00000000..6a47b1c4 --- /dev/null +++ b/scripts/expanded_prior_real_data_trial.py @@ -0,0 +1,368 @@ +"""Exercise corrected UltraNest expanded-prior warm starts on saved real light curves. + +This is a manual validation helper. It fits each data set with a deliberately +narrow Rp/R* prior, expands that prior, and compares the corrected warm-started +fit with an independent cold fit over the same expanded prior. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import math +import time +from pathlib import Path + +import numpy as np + +from exotic.api.elca import lc_fitter + + +DATASETS = ( + { + "name": "WASP-43b full transit", + "path": Path(r"D:\WASP43b_codex_current_fix_run\temp\NormalizedFlux_WASP-43 b_2026-03-20.txt"), + "format": "normalized", + "prior": { + "rprs": 0.1594, + "tmid": 2461120.68976, + "ars": 4.86, + "per": 0.813475, + "inc": 82.6, + "ecc": 0.0, + "omega": 90.0, + }, + "initial_rprs_bounds": [0.155, 0.165], + "expanded_rprs_bounds": [0.05, 0.30], + "tmid_bounds": [2461120.686, 2461120.694], + }, + { + "name": "TrES-5b full transit", + "path": Path( + r"D:\TrES5b_20260716_baron_rp\TrES5b_20260716_baron_rp" + r"\20260718_112527\Diagnostics\comp7\working_artifacts" + r"\FinalLightCurve_TrES-5b_2026-07-16.csv" + ), + "format": "final_lightcurve", + "prior": { + "rprs": 0.143, + "tmid": 2461238.83817, + "ars": 6.1, + "per": 1.48224686, + "inc": 84.27, + "ecc": 0.0, + "omega": 0.0, + }, + "initial_rprs_bounds": [0.140, 0.146], + "expanded_rprs_bounds": [0.05, 0.30], + "tmid_bounds": [2461238.834, 2461238.842], + }, + { + "name": "KELT-20b one-sided partial transit", + "path": Path( + r"D:\KELT-20\20260718_003832_codex_full_test" + r"\working_artifacts\NormalizedFlux_KELT-20b_2026-07-15.txt" + ), + "format": "normalized", + "prior": { + "rprs": 0.1144, + "tmid": 2461237.776, + "ars": 7.42, + "per": 3.4741085, + "inc": 86.12, + "ecc": 0.0, + "omega": 0.0, + }, + "initial_rprs_bounds": [0.110, 0.120], + "expanded_rprs_bounds": [0.02, 0.30], + "tmid_bounds": [2461237.736, 2461237.816], + }, +) + + +def load_normalized(path: Path) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + with path.open(newline="", encoding="utf-8-sig") as stream: + rows = list(csv.DictReader(stream)) + time_values = np.asarray([float(row["BJD"]) for row in rows], dtype=float) + flux = np.asarray([float(row["Norm Flux"]) for row in rows], dtype=float) + error = np.asarray([float(row["Norm Err"]) for row in rows], dtype=float) + airmass = np.asarray([float(row["AM"]) for row in rows], dtype=float) + return time_values, flux, error, airmass + + +def load_final_lightcurve(path: Path) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + values = np.loadtxt(path, delimiter=",", comments="#") + return values[:, 0], values[:, 2], values[:, 3], values[:, 5] + + +def load_dataset(config: dict) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + if config["format"] == "normalized": + return load_normalized(config["path"]) + return load_final_lightcurve(config["path"]) + + +def complete_prior(values: dict) -> dict: + return { + **values, + "u0": 0.0, + "u1": 0.0, + "u2": 0.0, + "u3": 0.0, + "a0": 1.0, + "a1": 1.0, + "a2": 0.0, + } + + +def summarize_fit(fit: lc_fitter, elapsed_seconds: float) -> dict: + samples = np.asarray(fit.results["samples"], dtype=float) + rprs_index = list(fit.sampled_keys).index("rprs") + rprs_samples = samples[:, rprs_index] + q16, median, q84 = np.quantile(rprs_samples, [0.16, 0.5, 0.84]) + results = fit.results + return { + "elapsed_seconds": elapsed_seconds, + "ncall": int(results.get("ncall", 0)), + "posterior_sample_count": int(samples.shape[0]), + "rprs_q16": float(q16), + "rprs_median": float(median), + "rprs_q84": float(q84), + "rprs_stdev": float(np.std(rprs_samples, ddof=1)), + "rprs_maximum_likelihood": float(fit.parameters["rprs"]), + "logz": float(results.get("logz", math.nan)), + "logzerr": float(results.get("logzerr", math.nan)), + "warmstart_attempted": bool( + getattr(fit, "ultranest_expanded_prior_warmstart_attempted", False) + ), + "warmstart_applied": bool( + getattr(fit, "ultranest_expanded_prior_warmstart_applied", False) + ), + "warmstart_note": getattr( + fit, "ultranest_expanded_prior_warmstart_note", None + ), + "warmstart_source_sample_count": int( + getattr( + fit, + "ultranest_expanded_prior_warmstart_source_sample_count", + 0, + ) + ), + "warmstart_source_effective_sample_size": float( + getattr( + fit, + "ultranest_expanded_prior_warmstart_effective_sample_size", + 0.0, + ) + ), + "warmstart_expanded_keys": list( + getattr( + fit, + "ultranest_expanded_prior_warmstart_expanded_keys", + [], + ) + ), + "warmstart_full_prior_fraction": float( + getattr( + fit, + "ultranest_expanded_prior_warmstart_full_prior_fraction", + math.nan, + ) + ), + } + + +def run_fit( + *, + time_values: np.ndarray, + flux: np.ndarray, + error: np.ndarray, + airmass: np.ndarray, + prior: dict, + bounds: list[float], + tmid_bounds: list[float], + seed: int, + live_points: int, + warmstart_source: lc_fitter | None = None, +) -> tuple[lc_fitter, dict]: + np.random.seed(seed) + started = time.perf_counter() + fit = lc_fitter( + time_values, + flux, + error, + airmass, + prior, + {"rprs": list(bounds), "tmid": list(tmid_bounds)}, + mode="ns", + jd_times=time_values, + verbose=False, + use_impactparameter_rather_than_inclination_to_fit=False, + ultranest_min_num_live_points=live_points, + ultranest_warmstart_source=warmstart_source, + ) + elapsed = time.perf_counter() - started + return fit, summarize_fit(fit, elapsed) + + +def run_dataset(config: dict, live_points: int, seed: int) -> dict: + time_values, flux, error, airmass = load_dataset(config) + finite = ( + np.isfinite(time_values) + & np.isfinite(flux) + & np.isfinite(error) + & np.isfinite(airmass) + & (error > 0) + ) + time_values = time_values[finite] + flux = flux[finite] + error = error[finite] + airmass = airmass[finite] + prior = complete_prior(config["prior"]) + + initial_fit, initial_summary = run_fit( + time_values=time_values, + flux=flux, + error=error, + airmass=airmass, + prior=prior, + bounds=config["initial_rprs_bounds"], + tmid_bounds=config["tmid_bounds"], + seed=seed, + live_points=live_points, + ) + warm_fit, warm_summary = run_fit( + time_values=time_values, + flux=flux, + error=error, + airmass=airmass, + prior=prior, + bounds=config["expanded_rprs_bounds"], + tmid_bounds=config["tmid_bounds"], + seed=seed + 1, + live_points=live_points, + warmstart_source=initial_fit, + ) + _, cold_summary = run_fit( + time_values=time_values, + flux=flux, + error=error, + airmass=airmass, + prior=prior, + bounds=config["expanded_rprs_bounds"], + tmid_bounds=config["tmid_bounds"], + seed=seed + 2, + live_points=live_points, + ) + + combined_sigma = math.hypot( + warm_summary["rprs_stdev"], cold_summary["rprs_stdev"] + ) + posterior_z = ( + abs(warm_summary["rprs_median"] - cold_summary["rprs_median"]) + / combined_sigma + if combined_sigma > 0 + else math.inf + ) + logz_sigma = math.hypot( + warm_summary["logzerr"], cold_summary["logzerr"] + ) + logz_z = ( + abs(warm_summary["logz"] - cold_summary["logz"]) / logz_sigma + if np.isfinite(logz_sigma) and logz_sigma > 0 + else math.nan + ) + old_lower, old_upper = config["initial_rprs_bounds"] + warm_outside_old = ( + warm_summary["rprs_median"] < old_lower + or warm_summary["rprs_median"] > old_upper + ) + cold_outside_old = ( + cold_summary["rprs_median"] < old_lower + or cold_summary["rprs_median"] > old_upper + ) + passed = ( + warm_summary["warmstart_applied"] + and warm_summary["warmstart_expanded_keys"] == ["rprs"] + and posterior_z <= 1.0 + and (not np.isfinite(logz_z) or logz_z <= 3.0) + and warm_outside_old == cold_outside_old + ) + return { + "name": config["name"], + "source_path": str(config["path"]), + "point_count": int(time_values.size), + "time_min_bjd_tdb": float(np.min(time_values)), + "time_max_bjd_tdb": float(np.max(time_values)), + "initial_rprs_bounds": list(config["initial_rprs_bounds"]), + "expanded_rprs_bounds": list(config["expanded_rprs_bounds"]), + "tmid_bounds": list(config["tmid_bounds"]), + "initial": initial_summary, + "expanded_warm": warm_summary, + "expanded_cold": cold_summary, + "comparison": { + "posterior_median_difference_sigma": float(posterior_z), + "logz_difference_sigma": float(logz_z), + "warm_median_outside_initial_bounds": bool(warm_outside_old), + "cold_median_outside_initial_bounds": bool(cold_outside_old), + }, + "passed": bool(passed), + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--live-points", type=int, default=100) + parser.add_argument("--seed", type=int, default=4729) + parser.add_argument( + "--dataset-index", + type=int, + action="append", + help="Zero-based dataset index to run; repeat to select more than one.", + ) + args = parser.parse_args() + + args.output.parent.mkdir(parents=True, exist_ok=True) + trial_started = time.perf_counter() + results = [] + selected_indices = ( + list(range(len(DATASETS))) + if args.dataset_index is None + else args.dataset_index + ) + for index in selected_indices: + if index < 0 or index >= len(DATASETS): + parser.error(f"--dataset-index must be between 0 and {len(DATASETS) - 1}") + config = DATASETS[index] + print(f"TRIAL START: {config['name']}", flush=True) + result = run_dataset( + config, + live_points=max(40, args.live_points), + seed=args.seed + index * 100, + ) + results.append(result) + print( + "TRIAL DONE: " + f"{config['name']} | pass={result['passed']} | " + f"warm={result['expanded_warm']['warmstart_applied']} | " + f"posterior_z={result['comparison']['posterior_median_difference_sigma']:.3f} | " + f"logz_z={result['comparison']['logz_difference_sigma']:.3f}", + flush=True, + ) + + payload = { + "live_points": max(40, args.live_points), + "seed": args.seed, + "elapsed_seconds": time.perf_counter() - trial_started, + "all_passed": all(result["passed"] for result in results), + "datasets": results, + } + args.output.write_text(json.dumps(payload, indent=2), encoding="utf-8") + print(f"RESULTS: {args.output}", flush=True) + print(f"ALL PASSED: {payload['all_passed']}", flush=True) + return 0 if payload["all_passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/setup.cfg b/setup.cfg index 19c0d3e3..2084197f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,7 +42,7 @@ install_requires = file: requirements.txt [options.entry_points] console_scripts = - exotic = exotic.exotic:main + exotic = exotic:cli exotic-gui = exotic.exotic_gui:main [options.packages.find] diff --git a/tests/test_aperture_tuning_performance.py b/tests/test_aperture_tuning_performance.py new file mode 100644 index 00000000..789f9005 --- /dev/null +++ b/tests/test_aperture_tuning_performance.py @@ -0,0 +1,173 @@ +import numpy as np +from astropy.io import fits + +import exotic.exotic as exotic_module + + +def test_evenly_spaced_aperture_tuning_indices_span_full_sequence(): + indices = exotic_module.evenly_spaced_aperture_tuning_indices(524, max_frames=24) + + assert len(indices) == 24 + assert indices[0] == 0 + assert indices[-1] == 523 + assert np.all(np.diff(indices) >= 22) + assert np.all(np.diff(indices) <= 23) + + +def test_centered_numpy_cutout_uses_local_slice_coordinates(): + image = np.arange(100, dtype=float).reshape(10, 10) + + cutout, local_x, local_y = exotic_module.centered_numpy_cutout( + image, + xc=5.25, + yc=4.75, + radius=2.0, + ) + + np.testing.assert_array_equal(cutout, image[2:8, 3:9]) + assert local_x == 2.25 + assert local_y == 2.75 + + +def test_memmap_cutouts_require_identity_frame_processing(): + empty = np.empty((0, 0)) + + assert exotic_module.can_memmap_aperture_tuning_cutouts( + generalDark=empty, + generalBias=empty, + generalFlat=empty, + demosaic_fmt=None, + bad_pixel_reference=None, + ) + assert not exotic_module.can_memmap_aperture_tuning_cutouts( + generalDark=np.ones((2, 2)), + ) + assert not exotic_module.can_memmap_aperture_tuning_cutouts( + demosaic_fmt="RGGB", + ) + assert not exotic_module.can_memmap_aperture_tuning_cutouts( + bad_pixel_reference={"coord_x": np.array([1]), "coord_y": np.array([1])}, + ) + + +def test_fits_header_memmap_guard_rejects_scaled_images(): + assert exotic_module.fits_header_supports_memmap({"BITPIX": -32}) + assert not exotic_module.fits_header_supports_memmap({"BSCALE": 2.0}) + assert not exotic_module.fits_header_supports_memmap({"BZERO": 32768}) + + +def test_missing_first_image_is_lazy_loaded_for_multiprocess_fov_plot(monkeypatch): + expected = np.arange(16, dtype=float).reshape(4, 4) + captured = {} + + def fake_load(file_name, *args, **kwargs): + captured["file_name"] = file_name + captured["bad_pixel_reference"] = kwargs["bad_pixel_reference"] + return expected + + monkeypatch.setattr(exotic_module, "load_calibrated_reduction_image", fake_load) + + retained = exotic_module.ensure_first_reduction_image_for_fov( + None, + "first.fits", + None, + None, + None, + None, + None, + None, + bad_pixel_reference="bad-pixel-map", + ) + + assert retained is expected + assert captured == { + "file_name": "first.fits", + "bad_pixel_reference": "bad-pixel-map", + } + + +def test_alignment_worker_memmap_path_skips_full_frame_calibration(tmp_path, monkeypatch): + frame_path = tmp_path / "frame.fits" + expected = np.arange(100, dtype=np.float32).reshape(10, 10) + fits.PrimaryHDU(expected).writeto(frame_path) + empty = np.empty((0, 0)) + exotic_module._ALIGNMENT_POOL_CONTEXT = { + "generalDark": empty, + "generalBias": empty, + "generalFlat": empty, + "demosaic_fmt": None, + "demosaic_out": None, + "demosaic_mult": None, + "bad_pixel_reference": None, + } + monkeypatch.setattr( + exotic_module, + "apply_cals", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("identity memmap path must not calibrate the full image") + ), + ) + + header, image = exotic_module._load_alignment_worker_frame(str(frame_path)) + + assert header["NAXIS1"] == 10 + np.testing.assert_array_equal(image[2:5, 3:7], expected[2:5, 3:7]) + + +def test_aperture_tuning_cutout_grid_matches_direct_local_measurement(): + y, x = np.indices((41, 41), dtype=float) + image = 100.0 + 5000.0 * np.exp(-((x - 20.2) ** 2 + (y - 19.8) ** 2) / (2.0 * 2.0 ** 2)) + frame = { + "frame_sigma": 2.0, + "stars": { + "comp1": { + "data": image, + "xc": 20.2, + "yc": 19.8, + "sigma": 2.0, + } + }, + } + apertures = np.array([2.5, 3.0]) + annuli = np.array([6.0, 8.0]) + + result = exotic_module.populate_aperture_tuning_data_from_cutouts( + [frame], + apertures, + annuli, + comparison_indices=(0,), + adaptive_apertures=True, + reference_sigma=2.0, + ) + expected_flux, expected_bg = exotic_module.compute_star_aperture_grid( + image, + 1, + 20.2, + 19.8, + apertures * 2.0, + annuli * 2.0, + sigma_hint=2.0, + ) + + np.testing.assert_allclose(result["comp1"][0], expected_flux) + np.testing.assert_allclose(result["comp1_bg"][0], expected_bg) + + +def test_image_process_pool_uses_spawn_context_on_windows(monkeypatch): + captured = {} + + class FakeProcessPool: + def __init__(self, *args, **kwargs): + captured["args"] = args + captured["kwargs"] = kwargs + + fake_context = object() + monkeypatch.setattr(exotic_module.sys, "platform", "win32") + monkeypatch.setattr(exotic_module.multiprocessing, "get_context", lambda mode: fake_context) + monkeypatch.setattr(exotic_module, "_ProcessPoolExecutor", FakeProcessPool) + + executor = exotic_module.ImageProcessPoolExecutor(max_workers=3) + + assert isinstance(executor, FakeProcessPool) + assert captured["kwargs"]["max_workers"] == 3 + assert captured["kwargs"]["mp_context"] is fake_context diff --git a/tests/test_boolean_config.py b/tests/test_boolean_config.py new file mode 100644 index 00000000..3f1b1d6e --- /dev/null +++ b/tests/test_boolean_config.py @@ -0,0 +1,63 @@ +import pytest + +import exotic.exotic as exotic_module + + +BOOLEAN_RUNTIME_PARSERS = ( + exotic_module.is_fast_aperture_mask_enabled, + exotic_module.is_comp_star_required, + exotic_module.is_target_driven_comp_selection_enabled, + exotic_module.should_skip_low_comparison_coverage_rejection, + exotic_module.should_fit_lightcurve_to_every_comparison_candidate, + exotic_module.should_use_automatic_optimal_calibration_selector, + exotic_module.should_use_ensemble_photometry_rather_than_single_comp, + exotic_module.should_use_ensemble_photometry_for_stellar_variability, + exotic_module.should_photometer_fortuitous_variables, + exotic_module.should_use_single_comparison_for_fortuitous_variables, + exotic_module.should_use_nextastro_vsx_cache_first, + exotic_module.should_use_sparse_posterior_live_point_retry, + exotic_module.should_run_fast_ultranest_before_final_run, + exotic_module.should_run_final_residual_rejection, + exotic_module.should_use_legacy_psf_flux_mode, + exotic_module.should_run_final_fit_phase_residual_clip, + exotic_module.should_pick_comparison_by_eebls_snr, + exotic_module.should_use_deviation_from_expected_transit_in_qc, + exotic_module.should_exit_at_first_qc_pass_solution, + exotic_module.should_restrict_rprs_range, + exotic_module.should_use_prior_rprs_when_posterior_pinned, + exotic_module.should_restrict_ars_range, + exotic_module.should_use_psf_photometry, + exotic_module.should_use_aperture_photometry, + exotic_module.should_use_eebls_to_initialize_tmid_and_bounds, + exotic_module.should_detect_bad_pixels_before_photometry, + exotic_module.is_adaptive_aperture_mode_enabled, + exotic_module.should_use_aperture_corrections_and_full_image_fwhm, + exotic_module.should_reject_overexposed_stars, + exotic_module.should_ignore_header_wcs, + exotic_module.should_prefer_pixel_values_over_wcs_for_target, + exotic_module.is_vertical_flux_normalization_disabled, + exotic_module.should_run_stellar_variability_only, + exotic_module.is_out_of_transit_baseline_detrending_enabled, + exotic_module.should_use_impactparameter_rather_than_inclination_to_fit, + exotic_module.should_require_apparent_magnitudes, + exotic_module.should_use_exactly_the_comps_provided, +) + + +TRUE_VARIANTS = (True, 1, "1", "y", "Y", "yes", "TRUE", "on") +FALSE_VARIANTS = (False, 0, "0", "n", "N", "no", "FALSE", "off") + + +@pytest.mark.parametrize("parser", BOOLEAN_RUNTIME_PARSERS, ids=lambda parser: parser.__name__) +def test_runtime_boolean_parser_accepts_every_supported_form(parser): + for value in TRUE_VARIANTS: + assert parser(value) is True + for value in FALSE_VARIANTS: + assert parser(value) is False + + +def test_multiprocess_bad_pixel_boolean_forms_are_consistent(): + for value in TRUE_VARIANTS: + assert exotic_module.get_multiprocess_bad_pixel_precheck_processes(value) >= 1 + for value in FALSE_VARIANTS: + assert exotic_module.get_multiprocess_bad_pixel_precheck_processes(value) is None diff --git a/tests/test_centroid_wcs.py b/tests/test_centroid_wcs.py new file mode 100644 index 00000000..2266558a --- /dev/null +++ b/tests/test_centroid_wcs.py @@ -0,0 +1,1526 @@ +import io +import sys +import types +import importlib.util +import threading +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +import pytest +from astropy.io import fits +from astropy.wcs import WCS + + +def _module_available(name): + try: + return importlib.util.find_spec(name) is not None + except (ModuleNotFoundError, ValueError): + return False + + +def _install_stub_module(name, **attrs): + module = types.ModuleType(name) + for attr, value in attrs.items(): + setattr(module, attr, value) + sys.modules[name] = module + return module + + +class _DummyDAOStarFinder: + def __init__(self, *args, **kwargs): + pass + + def __call__(self, *args, **kwargs): + return None + + +if not _module_available("astroalign"): + _install_stub_module("astroalign", PIXEL_TOL=1) +if not _module_available("astroquery"): + _install_stub_module("astroquery") +if not _module_available("astroquery.simbad"): + _install_stub_module("astroquery.simbad", Simbad=object) +if not _module_available("astroquery.gaia"): + _install_stub_module("astroquery.gaia", Gaia=object) +if not _module_available("barycorrpy"): + _install_stub_module("barycorrpy") +if not _module_available("barycorrpy.utc_tdb"): + _install_stub_module("barycorrpy.utc_tdb", JDUTC_to_BJDTDB=lambda *args, **kwargs: None) +if not _module_available("imreg_dft"): + _install_stub_module("imreg_dft") +if not _module_available("pyvo"): + _install_stub_module("pyvo") +if not _module_available("photutils"): + _install_stub_module("photutils") +if not _module_available("photutils.aperture"): + _install_stub_module("photutils.aperture", CircularAperture=object, CircularAnnulus=object) +if not _module_available("photutils.detection"): + _install_stub_module("photutils.detection", DAOStarFinder=_DummyDAOStarFinder) +if not _module_available("colour_demosaicing"): + _install_stub_module("colour_demosaicing", demosaicing_CFA_Bayer_bilinear=lambda *args, **kwargs: None) +if not _module_available("ldtk"): + fake_ldtk = _install_stub_module("ldtk") + fake_ldtk.LDPSet = type("LDPSet", (), {}) + fake_ldtk.ldtk = types.SimpleNamespace(LDPSet=fake_ldtk.LDPSet) +if not _module_available("ldtk.ldmodel"): + _install_stub_module( + "ldtk.ldmodel", + LinearModel=type("LinearModel", (), {}), + QuadraticModel=type("QuadraticModel", (), {}), + NonlinearModel=type("NonlinearModel", (), {}), + ) +if not _module_available("lmfit"): + _install_stub_module("lmfit") + +_install_stub_module( + "exotic.api.elca", + lc_fitter=lambda *args, **kwargs: None, + binner=lambda *args, **kwargs: None, + transit=lambda *args, **kwargs: None, + get_phase=lambda *args, **kwargs: None, +) + +from exotic import exotic as exotic_module + + +@pytest.mark.parametrize( + ("header_value", "expected"), + ( + ("120", 120.0), + ("120.0s", 120.0), + ("120.0 sec", 120.0), + ("120.0 secs", 120.0), + ("120.0 seconds", 120.0), + ("exposure 1.2e2 seconds", 120.0), + ), +) +def test_get_exp_time_accepts_numeric_strings_with_unit_text(header_value, expected): + assert exotic_module.get_exp_time({"EXPTIME": header_value}) == pytest.approx(expected) + + +def test_get_exp_time_rejects_unit_text_without_a_number(): + assert exotic_module.get_exp_time({"EXPTIME": "seconds"}) == 0.0 + + +def _gaussian_image(shape=(80, 80), center=(40.0, 35.0), amplitude=5000.0, sigma=2.0, background=100.0): + y, x = np.indices(shape, dtype=float) + cx, cy = center + image = background + amplitude * np.exp(-((x - cx) ** 2 + (y - cy) ** 2) / (2.0 * sigma ** 2)) + return image + + +def _write_extension_wcs_fits(tmp_path, shape=(100, 120)): + wcs = WCS(naxis=2) + wcs.wcs.crpix = [shape[1] / 2.0, shape[0] / 2.0] + wcs.wcs.crval = [210.0, 54.0] + wcs.wcs.cdelt = np.array([-0.01, 0.01]) + wcs.wcs.ctype = ["RA---TAN", "DEC--TAN"] + + path = tmp_path / "extension_wcs.fits" + hdul = fits.HDUList([ + fits.PrimaryHDU(), + fits.ImageHDU(data=np.zeros(shape, dtype=float), header=wcs.to_header(), name="SCI"), + ]) + hdul.writeto(path, overwrite=True) + return path + + +def test_detect_frame_bad_pixels_flags_isolated_hot_pixel_but_not_broad_star_core(): + image = _gaussian_image(shape=(60, 60), center=(30.0, 30.0), amplitude=1200.0, sigma=1.8, background=100.0) + image[10, 15] = 8000.0 + + mask = exotic_module.detect_frame_bad_pixels(image) + + assert mask[10, 15] + assert not mask[30, 30] + + +def test_build_persistent_bad_pixel_map_thresholds_recurrence_and_saves_outputs(tmp_path): + frames = {} + for frame_index in range(10): + frame = np.full((9, 9), 100.0, dtype=float) + if frame_index < 4: + frame[2, 3] = 4000.0 + if frame_index < 3: + frame[6, 5] = 3500.0 + frames[f"frame_{frame_index}.fits"] = frame + + reference = exotic_module.build_persistent_bad_pixel_map( + list(frames.keys()), + lambda file_name: frames[file_name], + save_directory=tmp_path, + ) + + assert reference is not None + assert reference["required_count"] == 4 + assert reference["mask"][2, 3] + assert not reference["mask"][6, 5] + + count_image = fits.getdata(tmp_path / "working_artifacts" / "BadPixelDetectionCounts.fits") + mask_image = fits.getdata(tmp_path / "working_artifacts" / "BadPixelMask.fits").astype(bool) + + assert count_image[2, 3] == 4 + assert count_image[6, 5] == 3 + assert mask_image[2, 3] + assert not mask_image[6, 5] + + +def test_build_persistent_bad_pixel_map_can_scan_with_multiprocessing(tmp_path, monkeypatch): + paths = [] + for frame_index in range(10): + frame = np.full((9, 9), 100.0, dtype=float) + if frame_index < 4: + frame[2, 3] = 4000.0 + path = tmp_path / f"frame_{frame_index}.fits" + fits.writeto(path, frame, overwrite=True) + paths.append(path) + + captured = {} + + class FakeFuture: + def __init__(self, value): + self._value = value + + def result(self): + return self._value + + class FakeExecutor: + def __init__(self, max_workers, initializer=None, initargs=()): + captured["max_workers"] = max_workers + if initializer is not None: + initializer(*initargs) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback): + return False + + def submit(self, fn, task): + return FakeFuture(fn(task)) + + monkeypatch.setattr(exotic_module, "ProcessPoolExecutor", FakeExecutor) + monkeypatch.setattr(exotic_module, "as_completed", lambda futures: futures) + + reference = exotic_module.build_persistent_bad_pixel_map( + paths, + exotic_module.load_image_data, + save_directory=tmp_path, + max_processes=2, + ) + + assert captured["max_workers"] == 2 + assert reference is not None + assert reference["required_count"] == 4 + assert reference["mask"][2, 3] + + +def test_repair_bad_pixels_in_frame_replaces_known_bad_pixel_with_neighbor_median(): + image = np.arange(25, dtype=float).reshape(5, 5) + image[2, 2] = 9999.0 + reference = { + "mask": np.zeros((5, 5), dtype=bool), + "coord_y": np.array([2]), + "coord_x": np.array([2]), + } + reference["mask"][2, 2] = True + + repaired = exotic_module.repair_bad_pixels_in_frame(image, reference) + + assert repaired[2, 2] == pytest.approx(12.0) + + +def test_fit_centroid_uses_moment_fallback_when_psf_fit_fails(monkeypatch): + image = _gaussian_image() + low_flux_warnings = [] + + def fail_least_squares(*args, **kwargs): + raise ValueError("Residuals are not finite in the initial point.") + + monkeypatch.setattr(exotic_module, "least_squares", fail_least_squares) + monkeypatch.setattr( + exotic_module.plateStatus, + "lowFluxAmplitudeWarning", + lambda star_index, xc, yc: low_flux_warnings.append((star_index, xc, yc)), + ) + + result = exotic_module.fit_centroid(image, [40.0, 35.0], 0) + + assert np.isfinite(result[0]) + assert np.isfinite(result[1]) + assert abs(result[0] - 40.0) < 1.0 + assert abs(result[1] - 35.0) < 1.0 + assert low_flux_warnings == [] + + +def test_fit_centroid_reports_consistent_background_between_fast_and_full_modes(): + image = _gaussian_image(center=(40.3, 35.7), amplitude=5000.0, sigma=2.0, background=123.4) + + fast_result = exotic_module.fit_centroid(image, [40.0, 36.0], 0, fast_mode=True) + full_result = exotic_module.fit_centroid(image, [40.0, 36.0], 0, fast_mode=False) + + assert np.isfinite(fast_result[6]) + assert np.isfinite(full_result[6]) + assert full_result[6] == pytest.approx(fast_result[6], abs=1e-8) + + +def test_fit_centroid_full_mode_preserves_psf_subpixel_solution(): + rng = np.random.default_rng(7) + true_center = (40.3, 35.7) + image = _gaussian_image( + center=true_center, + amplitude=120.0, + sigma=0.8, + background=1000.0, + ) + image += rng.normal(0.0, 20.0, size=image.shape) + + full_result = exotic_module.fit_centroid(image, [40.0, 36.0], 0, fast_mode=False) + psf_result = exotic_module.fit_centroid( + image, + [40.0, 36.0], + 0, + fast_mode=False, + weightedcenter=False, + ) + moment_result = exotic_module.fit_centroid(image, [40.0, 36.0], 0, fast_mode=True) + + assert full_result[0] == pytest.approx(psf_result[0], abs=1e-6) + assert full_result[1] == pytest.approx(psf_result[1], abs=1e-6) + + psf_error = np.hypot(psf_result[0] - true_center[0], psf_result[1] - true_center[1]) + moment_error = np.hypot(moment_result[0] - true_center[0], moment_result[1] - true_center[1]) + + assert psf_error < moment_error + + +def test_fit_psf_photometry_flux_row_preserves_robust_centroid_coordinates(): + image = _gaussian_image(center=(40.3, 35.7), amplitude=180.0, sigma=0.9, background=1000.0) + centroid_row = np.array([40.1, 35.9, 150.0, 0.8, 0.8, 0.0, 1000.0], dtype=float) + + flux_row = exotic_module.fit_psf_photometry_flux_row(image, centroid_row, 0) + + assert flux_row[0] == pytest.approx(centroid_row[0]) + assert flux_row[1] == pytest.approx(centroid_row[1]) + assert flux_row[2] > 0 + assert 0.5 <= flux_row[3] <= 2.0 + assert 0.5 <= flux_row[4] <= 2.0 + + +def test_fit_centroid_prefers_seed_anchored_solution_in_crowded_field(): + yy, xx = np.mgrid[0:80, 0:80] + image = np.full((80, 80), 400.0) + image += 80.0 * np.exp(-((xx - 40.0) ** 2 + (yy - 40.0) ** 2) / (2.0 * 1.0 ** 2)) + image += 220.0 * np.exp(-((xx - 31.5) ** 2 + (yy - 35.0) ** 2) / (2.0 * 1.0 ** 2)) + + result = exotic_module.fit_centroid(image, [40.0, 40.0], 0, fast_mode=False) + + assert np.hypot(result[0] - 40.0, result[1] - 40.0) < 1.5 + assert np.hypot(result[0] - 31.5, result[1] - 35.0) > 5.0 + assert result[2] > 0 + + +def test_fit_centroid_or_warn_out_of_frame_skips_centroid_fit(monkeypatch): + image = np.zeros((40, 50), dtype=float) + out_of_frame_warnings = [] + + monkeypatch.setattr( + exotic_module, + "fit_centroid", + lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("fit_centroid should be skipped")), + ) + monkeypatch.setattr( + exotic_module.plateStatus, + "outOfFrameWarning", + lambda star_index: out_of_frame_warnings.append(star_index), + ) + + result = exotic_module.fit_centroid_or_warn_out_of_frame(image, [75.0, 20.0], 1) + + assert np.all(np.isnan(result)) + assert out_of_frame_warnings == [1] + + +def test_skybg_phot_returns_nan_when_annulus_box_is_empty(monkeypatch): + image = np.zeros((20, 20), dtype=float) + sky_warnings = [] + + class _EmptyAnnulusMask: + data = np.empty((0, 0), dtype=float) + + def cutout(self, *args, **kwargs): + return None + + class _EmptyCircularAnnulus: + def __init__(self, *args, **kwargs): + pass + + def to_mask(self, *args, **kwargs): + return [_EmptyAnnulusMask()] + + monkeypatch.setattr(exotic_module, "CircularAnnulus", _EmptyCircularAnnulus) + monkeypatch.setattr( + exotic_module.plateStatus, + "skyBackgroundWarning", + lambda star_index, xc, yc: sky_warnings.append((star_index, xc, yc)), + ) + + bgflux, sigmabg, nbg = exotic_module.skybg_phot(image, 0, 30.0, 30.0) + + assert np.isnan(bgflux) + assert np.isnan(sigmabg) + assert nbg == 0 + assert sky_warnings == [(0, 30.0, 30.0)] + + +@pytest.mark.skipif(not _module_available("photutils.aperture"), reason="requires photutils aperture masks") +def test_skybg_phot_exact_annulus_uses_fractional_pixel_area(): + image = np.ones((80, 80), dtype=float) + + bgflux, sigmabg, nbg = exotic_module.skybg_phot(image, 0, 40.3, 35.7, r=3.0, dr=2.0, fast_mode=False) + + assert bgflux == pytest.approx(1.0, abs=1e-8) + assert sigmabg == pytest.approx(0.0, abs=1e-8) + assert nbg == pytest.approx(np.pi * ((3.0 + 2.0) ** 2 - 3.0 ** 2), rel=1e-3) + assert not np.isclose(nbg, round(nbg), atol=1e-6) + + +@pytest.mark.skipif(not _module_available("photutils.aperture"), reason="requires photutils aperture masks") +def test_skybg_phot_high_side_clipping_rejects_hot_pixel(): + image = np.full((80, 80), 100.0, dtype=float) + image[40, 55] = 10000.0 + + bgflux, sigmabg, nbg = exotic_module.skybg_phot(image, 0, 40.0, 40.0, r=10.0, dr=10.0, fast_mode=False) + + assert bgflux == pytest.approx(100.0, abs=1e-8) + assert sigmabg == pytest.approx(0.0, abs=1e-8) + assert nbg > 250.0 + + +def test_check_target_pixel_wcs_keeps_input_coords_when_wcs_target_is_off_frame(monkeypatch): + image = np.zeros((100, 120), dtype=float) + wcs = WCS(naxis=2) + wcs.wcs.crpix = [60.0, 50.0] + wcs.wcs.crval = [210.0, 54.0] + wcs.wcs.cdelt = np.array([-0.01, 0.01]) + wcs.wcs.ctype = ["RA---TAN", "DEC--TAN"] + header = wcs.to_header() + header["NAXIS"] = 2 + header["NAXIS1"] = 120 + header["NAXIS2"] = 100 + + ra_list, dec_list = exotic_module.get_ra_dec(header) + + monkeypatch.setattr( + exotic_module, + "update_coordinates_with_proper_motion", + lambda info_dict, obs_time: (212.0, 54.0), + ) + monkeypatch.setattr( + exotic_module, + "get_psf_parameters", + lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("centroiding should be skipped")), + ) + + x_pixel, y_pixel = exotic_module.check_target_pixel_wcs( + 25.0, + 30.0, + {"ra": 210.0, "dec": 54.0, "dist": 0.0, "pm_ra": 0.0, "pm_dec": 0.0}, + ra_list, + dec_list, + image, + 2461100.5, + non_interactive_run=True, + wcs_header=header, + ) + + assert x_pixel == 25.0 + assert y_pixel == 30.0 + + +def test_get_ra_dec_uses_image_shape_when_header_lacks_naxis(): + wcs = WCS(naxis=2) + wcs.wcs.crpix = [60.0, 50.0] + wcs.wcs.crval = [210.0, 54.0] + wcs.wcs.cdelt = np.array([-0.01, 0.01]) + wcs.wcs.ctype = ["RA---TAN", "DEC--TAN"] + + ra_list, dec_list = exotic_module.get_ra_dec(wcs.to_header(), image_shape=(100, 120)) + + assert ra_list.shape == (100, 120) + assert dec_list.shape == (100, 120) + + +def test_get_ra_dec_matches_zero_based_astropy_pixel_coordinates(): + wcs = WCS(naxis=2) + wcs.wcs.crpix = [60.0, 50.0] + wcs.wcs.crval = [210.0, 54.0] + wcs.wcs.cdelt = np.array([-0.01, 0.01]) + wcs.wcs.ctype = ["RA---TAN", "DEC--TAN"] + header = wcs.to_header() + header["NAXIS"] = 2 + header["NAXIS1"] = 120 + header["NAXIS2"] = 100 + + ra_list, dec_list = exotic_module.get_ra_dec(header) + + for x_pixel, y_pixel in [(0, 0), (59, 49), (119, 99), (23, 71)]: + expected_ra, expected_dec = wcs.pixel_to_world_values(x_pixel, y_pixel) + assert ra_list[y_pixel, x_pixel] == pytest.approx(expected_ra, abs=1.0e-12) + assert dec_list[y_pixel, x_pixel] == pytest.approx(expected_dec, abs=1.0e-12) + + +def test_get_first_image_header_skips_empty_primary_hdu(tmp_path): + wcs_path = _write_extension_wcs_fits(tmp_path) + + header = exotic_module.get_first_image_header(wcs_path) + + assert header["NAXIS1"] == 120 + assert header["NAXIS2"] == 100 + assert header["CTYPE1"] == "RA---TAN" + + +def test_get_img_scale_uses_first_image_extension_header(tmp_path): + wcs_path = _write_extension_wcs_fits(tmp_path) + + img_scale_str, img_scale = exotic_module.get_img_scale(fits.Header(), wcs_path, None) + + assert img_scale_str == "Image scale in arcsec/pixel: 36.0" + assert img_scale == 36.0 + + +def test_should_ignore_header_wcs_defaults_to_false(): + assert exotic_module.should_ignore_header_wcs(None) is False + for value in (True, 1, "1", "y", "Y", "yes", "TRUE", "on"): + assert exotic_module.should_ignore_header_wcs(value) is True + for value in (False, 0, "0", "n", "N", "no", "FALSE", "off"): + assert exotic_module.should_ignore_header_wcs(value) is False + + +def test_should_allow_pixel_alignment_fallback_defaults_to_true(): + assert exotic_module.should_allow_pixel_alignment_fallback(None) is True + for value in (True, 1, "1", "y", "Y", "yes", "TRUE", "on"): + assert exotic_module.should_allow_pixel_alignment_fallback(value) is True + for value in (False, 0, "0", "n", "N", "no", "FALSE", "off"): + assert exotic_module.should_allow_pixel_alignment_fallback(value) is False + + +def test_get_bad_wcs_threshold_fraction_defaults_to_three_percent(): + assert exotic_module.get_bad_wcs_threshold_fraction(None) == pytest.approx(0.03) + assert exotic_module.get_bad_wcs_threshold_fraction("") == pytest.approx(0.03) + + +def test_get_bad_wcs_threshold_fraction_reads_numeric_percent_values(): + assert exotic_module.get_bad_wcs_threshold_fraction(5.5) == pytest.approx(0.055) + assert exotic_module.get_bad_wcs_threshold_fraction("7.25") == pytest.approx(0.0725) + assert exotic_module.get_bad_wcs_threshold_fraction("4%") == pytest.approx(0.04) + + +def test_get_bad_wcs_threshold_fraction_falls_back_for_invalid_values(): + assert exotic_module.get_bad_wcs_threshold_fraction("not-a-number") == pytest.approx(0.03) + assert exotic_module.get_bad_wcs_threshold_fraction(-1) == pytest.approx(0.03) + assert exotic_module.get_bad_wcs_threshold_fraction(101) == pytest.approx(0.03) + + +def test_get_pointing_rejection_sigma_defaults_to_disabled(): + assert exotic_module.get_pointing_rejection_sigma(None) is None + assert exotic_module.get_pointing_rejection_sigma("") is None + + +def test_get_pointing_rejection_sigma_reads_positive_numeric_values(): + assert exotic_module.get_pointing_rejection_sigma(3) == pytest.approx(3.0) + assert exotic_module.get_pointing_rejection_sigma("2.75") == pytest.approx(2.75) + + +def test_get_pointing_rejection_sigma_disables_for_invalid_text_and_zero(): + assert exotic_module.get_pointing_rejection_sigma("not-a-number") is None + assert exotic_module.get_pointing_rejection_sigma(-1) is None + assert exotic_module.get_pointing_rejection_sigma(0) is None + assert exotic_module.get_pointing_rejection_sigma("off") is None + + +def test_display_filename_returns_basename_for_unix_and_windows_paths(): + assert ( + exotic_module._display_filename( + "/content/drive/MyDrive/0.Exoplanets/2.Transits/run/frame_001.fits.fz" + ) + == "frame_001.fits.fz" + ) + assert exotic_module._display_filename(r"C:\data\run\frame_002.fits.fz") == "frame_002.fits.fz" + + +def test_format_plate_solution_reference_uses_basename_only(): + assert ( + exotic_module.format_plate_solution_reference( + "/mnt/md0/ftp/user_data/psyfitz/DATA_INBOX/Z.good.TOI 2969 b_2026-03-05_ECO1/" + "NxAst-TOI2969b_rp_2461105d05262731_20260305_1a016_30_eco1.fits.fz" + ) + == "Here is the filename where we got the WCS from: " + "NxAst-TOI2969b_rp_2461105d05262731_20260305_1a016_30_eco1.fits.fz" + ) + + +def test_log_alignment_progress_reports_wcs_location_and_basename(monkeypatch): + stdout = io.StringIO() + debug_messages = [] + + monkeypatch.setattr(exotic_module.sys, "stdout", stdout) + monkeypatch.setattr(exotic_module.log, "debug", lambda message: debug_messages.append(message)) + + exotic_module.log_alignment_progress( + 144, + 220, + "/content/drive/MyDrive/0.Exoplanets/2.Transits/run/frame_145.fits.fz", + False, + ) + + expected = "WCS-locating stars in frame 145 of 220 : frame_145.fits.fz\n" + assert stdout.getvalue() == expected + assert debug_messages == [expected] + + +def test_log_alignment_progress_only_says_pixel_aligning_when_enabled(monkeypatch): + stdout = io.StringIO() + monkeypatch.setattr(exotic_module.sys, "stdout", stdout) + monkeypatch.setattr(exotic_module.log, "debug", lambda _message: None) + + exotic_module.log_alignment_progress( + 0, + 2, + "frame_1.fits", + False, + pixel_alignment_enabled=True, + ) + + assert stdout.getvalue() == "Pixel-aligning frame 1 of 2 : frame_1.fits\n" + + +def test_collect_transform_frame_pointings_logs_alignment_progress(monkeypatch): + progress_messages = [] + + monkeypatch.setattr( + exotic_module, + "log_info", + lambda message, warn=False, error=False: progress_messages.append((message, warn, error)), + ) + monkeypatch.setattr( + exotic_module, + "transformation", + lambda image_data, file_name, report_failure=False, reference_image=None: ( + lambda anchor: np.asarray(anchor, dtype=float) + ), + ) + + frames = ["frame_0001.fits", "frame_0002.fits", "frame_0003.fits"] + frame_loader = lambda file_name: np.ones((8, 8), dtype=float) + + positions, usable_mask = exotic_module.collect_transform_frame_pointings(frames, frame_loader=frame_loader) + + assert positions.shape == (3, 2) + assert usable_mask.tolist() == [True, True, True] + assert [message for message, _, _ in progress_messages] == [ + "Pointing precheck alignment progress: file 1 of 3 : frame_0001.fits", + "Pointing precheck alignment progress: file 2 of 3 : frame_0002.fits", + "Pointing precheck alignment progress: file 3 of 3 : frame_0003.fits", + ] + + +def test_collect_transform_frame_pointings_can_return_transform_cache(monkeypatch): + monkeypatch.setattr( + exotic_module, + "log_info", + lambda *_args, **_kwargs: None, + ) + + expected_transform = exotic_module.SimilarityTransform(scale=1, rotation=0, translation=[2.0, -1.0]) + monkeypatch.setattr( + exotic_module, + "transformation", + lambda image_data, file_name, report_failure=False, reference_image=None: expected_transform, + ) + + frames = ["frame_0001.fits", "frame_0002.fits"] + frame_loader = lambda file_name: np.ones((8, 8), dtype=float) + + positions, usable_mask, transforms = exotic_module.collect_transform_frame_pointings( + frames, + frame_loader=frame_loader, + return_transforms=True, + ) + + assert usable_mask.tolist() == [True, True] + assert np.allclose(positions[1], [5.5, 2.5]) + assert set(transforms) == set(frames) + assert transforms[frames[1]] is expected_transform + + +def test_check_wcs_ignores_header_wcs_when_override_enabled(monkeypatch): + monkeypatch.setattr( + exotic_module, + "search_wcs", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("header WCS should be ignored")), + ) + + wcs_file = exotic_module.check_wcs( + "frame.fits", + ".", + "n", + ignore_header_wcs=True, + ) + + assert wcs_file is None + + +def test_check_wcs_keeps_plate_solution_when_override_enabled(monkeypatch): + monkeypatch.setattr(exotic_module, "get_wcs", lambda *_args, **_kwargs: "solved_wcs.fits") + monkeypatch.setattr( + exotic_module, + "search_wcs", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("header WCS should not be consulted")), + ) + + wcs_file = exotic_module.check_wcs( + "frame.fits", + ".", + "y", + ignore_header_wcs=True, + ) + + assert wcs_file == "solved_wcs.fits" + + +def test_check_wcs_prefers_header_wcs_over_plate_solution(monkeypatch): + monkeypatch.setattr( + exotic_module, + "search_wcs", + lambda *_args, **_kwargs: types.SimpleNamespace(is_celestial=True), + ) + monkeypatch.setattr( + exotic_module, + "get_wcs", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("plate solution should be skipped when header WCS exists") + ), + ) + + wcs_file = exotic_module.check_wcs( + "frame.fits", + ".", + "y", + ) + + assert wcs_file == "frame.fits" + + +def test_should_log_plate_solution_path_suppresses_posix_tmp_paths(): + assert exotic_module.should_log_plate_solution_path("/tmp/tmp42fmzrv8/temp/wcs.fits") is False + + +def test_should_log_plate_solution_path_keeps_non_tmp_paths(): + assert exotic_module.should_log_plate_solution_path("/data/session/temp/wcs.fits") is True + assert exotic_module.should_log_plate_solution_path("/tmp_backup/temp/wcs.fits") is True + + +def test_should_use_multiprocess_transform_precompute_respects_header_wcs_override(): + assert exotic_module.should_use_multiprocess_transform_precompute( + ["frame1.fits", "frame2.fits"], + requested_processes=2, + ignore_header_wcs=True, + ) is True + + +def test_transformation_pool_initializer_suppresses_inherited_tk_cleanup(monkeypatch): + calls = [] + reference_image = np.ones((4, 4), dtype=float) + monkeypatch.setattr(exotic_module, "suppress_inherited_tk_cleanup_in_worker", lambda: calls.append(True)) + monkeypatch.setattr(exotic_module, "load_image_data", lambda _file_name: reference_image) + + exotic_module._TRANSFORM_REFERENCE_IMAGE = None + exotic_module._TRANSFORM_REFERENCE_CACHE = {"stale": True} + + exotic_module._transformation_pool_initializer("reference.fits") + + assert calls == [True] + assert exotic_module._TRANSFORM_REFERENCE_IMAGE is reference_image + assert exotic_module._TRANSFORM_REFERENCE_CACHE is None + + +def test_apply_parallel_alignment_result_uses_precomputed_fallback_when_wcs_geometry_fails(monkeypatch): + psf_data = { + "target": np.zeros((2, 7), dtype=float), + "comp1": np.zeros((2, 7), dtype=float), + } + psf_data["target"][0] = np.array([10.0, 10.0, 100.0, 2.0, 2.0, 0.0, 50.0]) + psf_data["comp1"][0] = np.array([20.0, 10.0, 100.0, 2.0, 2.0, 0.0, 50.0]) + tar_comp_dist = {"comp1": np.array([10, 0], dtype=int)} + warnings = [] + + monkeypatch.setattr( + exotic_module.plateStatus, + "lowFluxAmplitudeWarning", + lambda star_index, xc, yc: warnings.append((star_index, xc, yc)), + ) + + result = { + "index": 1, + "file_name": "frame_0002.fits", + "wcs": { + "projected_off_frame": False, + "psf_rows": { + "target": np.array([10.0, 10.0, 100.0, 2.0, 2.0, 0.0, 50.0]), + "comp1": np.array([50.0, 50.0, 100.0, 2.0, 2.0, 0.0, 50.0]), + }, + "warnings": [("low_flux", 1, 50.0, 50.0)], + }, + "fallback": { + "psf_rows": { + "target": np.array([11.0, 10.0, 100.0, 2.0, 2.0, 0.0, 50.0]), + "comp1": np.array([21.0, 10.0, 90.0, 2.0, 2.0, 0.0, 50.0]), + }, + "warnings": [("low_flux", 1, 21.0, 10.0)], + }, + } + + selected = exotic_module.apply_parallel_alignment_result( + result, + 1, + psf_data, + tar_comp_dist, + ["comp1"], + ) + + assert selected == "fallback" + assert psf_data["target"][1, 0] == pytest.approx(11.0) + assert psf_data["comp1"][1, 0] == pytest.approx(21.0) + assert warnings == [(1, 21.0, 10.0)] + + +def test_parallel_alignment_task_uses_precomputed_fallback_transform(monkeypatch): + target_and_comp_pixels = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=float) + precomputed_transform = exotic_module.SimilarityTransform( + scale=1, + rotation=0, + translation=[5.0, -1.0], + ) + + monkeypatch.setattr( + exotic_module, + "_load_alignment_worker_frame", + lambda _file_name: ({}, np.ones((10, 10), dtype=float)), + ) + monkeypatch.setattr( + exotic_module, + "transformation", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("cached transform should be used") + ), + ) + monkeypatch.setattr( + exotic_module, + "_fit_alignment_candidate_psfs", + lambda image_data, predicted_coords, *_args: { + "coords": np.asarray(predicted_coords, dtype=float), + "psf_rows": {"target": np.zeros(7, dtype=float)}, + "warnings": [], + }, + ) + + result = exotic_module._parallel_alignment_task(( + 1, + "frame_0002.fits", + target_and_comp_pixels, + None, + True, + False, + False, + True, + False, + precomputed_transform, + )) + + assert np.allclose(result["fallback"]["coords"], [[6.0, 1.0], [8.0, 3.0]]) + + +def test_classify_wcs_fallback_frames_queues_only_missing_or_rejected_candidates(): + def candidate(target_xy, comp_xy): + coords = np.array([target_xy, comp_xy], dtype=float) + return { + "coords": coords, + "projected_off_frame": False, + "psf_rows": { + "target": np.array([*target_xy, 100.0, 2.0, 2.0, 0.0, 50.0]), + "comp1": np.array([*comp_xy, 90.0, 2.0, 2.0, 0.0, 50.0]), + }, + "warnings": [], + } + + results = [ + {"index": 0, "file_name": "frame0.fits", "wcs": candidate((10.0, 10.0), (20.0, 10.0)), "fallback": None}, + {"index": 1, "file_name": "frame1.fits", "wcs": candidate((11.0, 10.0), (50.0, 50.0)), "fallback": None}, + {"index": 2, "file_name": "frame2.fits", "wcs": None, "fallback": None}, + {"index": 3, "file_name": "frame3.fits", "wcs": candidate((13.0, 10.0), (23.0, 10.0)), "fallback": None}, + ] + + missing, rejected = exotic_module.classify_wcs_fallback_frames( + results, + np.array([[10.0, 10.0], [20.0, 10.0]]), + ) + + assert missing == [2] + assert rejected == [1] + + +def test_build_multiprocess_alignment_results_runs_legacy_batch_only_for_wcs_failures(monkeypatch): + def candidate(target_xy, comp_xy): + coords = np.array([target_xy, comp_xy], dtype=float) + return { + "coords": coords, + "projected_off_frame": False, + "psf_rows": { + "target": np.array([*target_xy, 100.0, 2.0, 2.0, 0.0, 50.0]), + "comp1": np.array([*comp_xy, 90.0, 2.0, 2.0, 0.0, 50.0]), + }, + "warnings": [], + } + + batches = [] + + def fake_run_batch(tasks, *_args, **_kwargs): + batches.append(tasks) + if len(batches) == 1: + return { + 0: {"index": 0, "file_name": "frame0.fits", "wcs": candidate((10.0, 10.0), (20.0, 10.0)), "fallback": None}, + 1: {"index": 1, "file_name": "frame1.fits", "wcs": candidate((11.0, 10.0), (21.0, 10.0)), "fallback": None}, + 2: {"index": 2, "file_name": "frame2.fits", "wcs": candidate((12.0, 10.0), (50.0, 50.0)), "fallback": None}, + 3: {"index": 3, "file_name": "frame3.fits", "wcs": None, "fallback": None}, + } + + return { + task[0]: { + "index": task[0], + "file_name": task[1], + "wcs": None, + "fallback": candidate((10.0 + task[0], 10.0), (20.0 + task[0], 10.0)), + } + for task in tasks + } + + monkeypatch.setattr(exotic_module, "_run_multiprocess_alignment_task_batch", fake_run_batch) + + results = exotic_module.build_multiprocess_alignment_results( + np.array(["frame0.fits", "frame1.fits", "frame2.fits", "frame3.fits"]), + 4, + np.array([[10.0, 10.0], [20.0, 10.0]]), + target_and_comp_radec=np.array([[1.0, 2.0], [1.1, 2.1]]), + compute_fallback_transform=True, + ) + + assert [task[0] for task in batches[0]] == [0, 1, 2, 3] + assert all(task[7] is False for task in batches[0]) + assert [task[0] for task in batches[1]] == [2, 3] + assert all(task[4] is True and task[7] is True for task in batches[1]) + assert results[0]["fallback"] is None + assert results[1]["fallback"] is None + assert results[2]["fallback"] is not None + assert results[3]["fallback"] is not None + + +def test_build_multiprocess_alignment_results_does_not_fallback_by_default(monkeypatch): + batches = [] + + def fake_run_batch(tasks, *_args, **_kwargs): + batches.append(tasks) + return { + task[0]: { + "index": task[0], + "file_name": task[1], + "wcs": None, + "fallback": None, + } + for task in tasks + } + + monkeypatch.setattr(exotic_module, "_run_multiprocess_alignment_task_batch", fake_run_batch) + + results = exotic_module.build_multiprocess_alignment_results( + np.array(["frame0.fits", "frame1.fits"]), + 2, + np.array([[10.0, 10.0], [20.0, 10.0]]), + target_and_comp_radec=np.array([[1.0, 2.0], [1.1, 2.1]]), + ) + + assert len(batches) == 1 + assert all(task[7] is False for task in batches[0]) + assert all(result["fallback"] is None for result in results) + + +def test_parallel_wcs_task_uses_first_frames_own_wcs_projection(monkeypatch): + projected = np.array([[101.25, 202.5], [303.75, 404.5]], dtype=float) + captured = {} + + class FakeWcs: + is_celestial = True + + @staticmethod + def world_to_pixel_values(_ra, _dec): + return projected[:, 0], projected[:, 1] + + monkeypatch.setattr( + exotic_module, + "_load_alignment_worker_frame", + lambda _file_name: (object(), np.ones((512, 512), dtype=float)), + ) + monkeypatch.setattr(exotic_module, "search_wcs_from_header", lambda _header: FakeWcs()) + + def fake_fit(_image, predicted_coords, *_args, **_kwargs): + captured["coords"] = np.array(predicted_coords, dtype=float, copy=True) + return { + "coords": np.array(predicted_coords, dtype=float, copy=True), + "psf_rows": { + "target": np.array([101.25, 202.5, 1.0, 1.0, 1.0, 0.0, 0.0]), + "comp1": np.array([303.75, 404.5, 1.0, 1.0, 1.0, 0.0, 0.0]), + }, + "warnings": [], + } + + monkeypatch.setattr(exotic_module, "_fit_alignment_candidate_psfs", fake_fit) + + exotic_module._parallel_alignment_task(( + 0, + "frame0.fits", + np.array([[10.0, 20.0], [30.0, 40.0]]), + np.array([[1.0, 2.0], [1.1, 2.1]]), + False, + False, + False, + False, + True, + None, + )) + + np.testing.assert_allclose(captured["coords"], projected) + + +def test_downsampled_fallback_transformation_restores_full_resolution_translation(monkeypatch): + calls = [] + + def fake_transformation(image_data, _file_name, **kwargs): + calls.append((image_data.shape, kwargs["reference_image"].shape)) + return exotic_module.SimilarityTransform( + scale=1.01, + rotation=0.02, + translation=[2.0, -3.0], + ) + + monkeypatch.setattr(exotic_module, "transformation", fake_transformation) + image = np.ones((8, 12), dtype=float) + reference = np.ones((8, 12), dtype=float) + + result = exotic_module.downsampled_fallback_transformation( + image, + "frame.fits", + reference_image=reference, + max_dimension=6, + ) + + assert calls == [((4, 6), (4, 6))] + assert result.scale == pytest.approx(1.01) + assert result.rotation == pytest.approx(0.02) + assert np.allclose(result.translation, [4.0, -6.0]) + + +def test_fit_alignment_candidate_psfs_serializes_plate_status_swap(monkeypatch): + sentinel_status = types.SimpleNamespace(name="original-plate-status") + started = threading.Event() + + def fake_fit_centroid(_data, pos, starIndex, **_kwargs): + started.set() + return np.array([float(starIndex), float(pos[0]), float(pos[1])], dtype=float) + + monkeypatch.setattr(exotic_module, "plateStatus", sentinel_status) + monkeypatch.setattr(exotic_module, "fit_centroid_or_warn_out_of_frame", fake_fit_centroid) + + lock = exotic_module._PLATE_STATUS_SWAP_LOCK + lock.acquire() + executor = ThreadPoolExecutor(max_workers=1) + future = None + try: + future = executor.submit( + exotic_module._fit_alignment_candidate_psfs, + np.ones((5, 5), dtype=float), + np.array([[1.0, 1.0], [2.0, 2.0]], dtype=float), + False, + False, + ) + assert not started.wait(0.2) + finally: + lock.release() + + try: + result = future.result(timeout=2) + finally: + executor.shutdown(wait=True) + + assert started.wait(0.2) + assert result["psf_rows"]["target"].tolist() == [0.0, 1.0, 1.0] + assert result["psf_rows"]["comp1"].tolist() == [1.0, 2.0, 2.0] + assert exotic_module.plateStatus is sentinel_status + + +def test_filter_sparse_missing_wcs_frames_drops_files_below_three_percent(monkeypatch): + frames = [f"frame_{i}.fits" for i in range(34)] + missing_frame = frames[7] + + monkeypatch.setattr(exotic_module, "get_first_image_header", lambda file_name: str(file_name)) + monkeypatch.setattr( + exotic_module, + "search_wcs_from_header", + lambda header: types.SimpleNamespace(is_celestial=header != missing_frame), + ) + + filtered, keep_mask, dropped = exotic_module.filter_sparse_missing_wcs_frames(frames) + + assert filtered.tolist() == [frame for frame in frames if frame != missing_frame] + assert keep_mask.tolist() == [frame != missing_frame for frame in frames] + assert dropped == [missing_frame] + + +def test_filter_sparse_missing_wcs_frames_keeps_full_sequence_when_most_frames_lack_wcs(monkeypatch): + frames = [f"frame_{i}.fits" for i in range(46)] + only_wcs_frame = frames[0] + + monkeypatch.setattr(exotic_module, "get_first_image_header", lambda file_name: str(file_name)) + monkeypatch.setattr( + exotic_module, + "search_wcs_from_header", + lambda header: types.SimpleNamespace(is_celestial=header == only_wcs_frame), + ) + + filtered, keep_mask, dropped = exotic_module.filter_sparse_missing_wcs_frames(frames) + + assert filtered.tolist() == frames + assert keep_mask.tolist() == [True] * len(frames) + assert dropped == [] + + +def test_filter_sparse_missing_wcs_frames_can_drop_missing_wcs_when_pixel_fallback_is_disabled(monkeypatch): + frames = [f"frame_{i}.fits" for i in range(33)] + missing_frame = frames[5] + + monkeypatch.setattr(exotic_module, "get_first_image_header", lambda file_name: str(file_name)) + monkeypatch.setattr( + exotic_module, + "search_wcs_from_header", + lambda header: types.SimpleNamespace(is_celestial=header != missing_frame), + ) + + filtered, keep_mask, dropped = exotic_module.filter_sparse_missing_wcs_frames( + frames, + allow_pixel_alignment_fallback=False, + ) + + assert filtered.tolist() == [frame for frame in frames if frame != missing_frame] + assert keep_mask.tolist() == [frame != missing_frame for frame in frames] + assert dropped == [missing_frame] + + +def test_filter_wcs_target_out_of_frame_frames_drops_only_projected_misses(monkeypatch): + def make_wcs_header(center_ra): + wcs = WCS(naxis=2) + wcs.wcs.crpix = [60.0, 50.0] + wcs.wcs.crval = [center_ra, 54.0] + wcs.wcs.cdelt = np.array([-0.01, 0.01]) + wcs.wcs.ctype = ["RA---TAN", "DEC--TAN"] + header = wcs.to_header() + header["NAXIS"] = 2 + header["NAXIS1"] = 120 + header["NAXIS2"] = 100 + return header + + no_wcs_header = fits.Header() + no_wcs_header["NAXIS"] = 2 + no_wcs_header["NAXIS1"] = 120 + no_wcs_header["NAXIS2"] = 100 + + headers = { + "target_in_frame.fits": make_wcs_header(210.0), + "target_off_frame.fits": make_wcs_header(212.0), + "no_wcs.fits": no_wcs_header, + } + messages = [] + + monkeypatch.setattr(exotic_module, "get_first_image_header", lambda file_name: headers[file_name]) + monkeypatch.setattr( + exotic_module, + "update_coordinates_with_proper_motion", + lambda info_dict, obs_time: (210.0, 54.0), + ) + monkeypatch.setattr( + exotic_module, + "log_info", + lambda message, warn=False, error=False: messages.append((message, warn, error)), + ) + + frames = list(headers) + filtered, keep_mask, dropped = exotic_module.filter_wcs_target_out_of_frame_frames( + frames, + {"ra": 210.0, "dec": 54.0}, + obs_times=[2461196.5, 2461196.6, 2461196.7], + ) + + assert filtered.tolist() == ["target_in_frame.fits", "no_wcs.fits"] + assert keep_mask.tolist() == [True, False, True] + assert dropped == ["target_off_frame.fits"] + assert any("Target WCS precheck" in message for message, _, _ in messages) + + +def test_maybe_reinterpret_decimal_ra_hours_from_wcs_when_only_ra_times_fifteen_matches(monkeypatch): + wcs = WCS(naxis=2) + wcs.wcs.crpix = [60.0, 50.0] + wcs.wcs.crval = [16.18494, 74.3313] + wcs.wcs.cdelt = np.array([-0.01, 0.01]) + wcs.wcs.ctype = ["RA---TAN", "DEC--TAN"] + header = wcs.to_header() + header["NAXIS"] = 2 + header["NAXIS1"] = 120 + header["NAXIS2"] = 100 + messages = [] + + monkeypatch.setattr(exotic_module, "get_first_image_header", lambda _file_name: header) + monkeypatch.setattr( + exotic_module, + "log_info", + lambda message, warn=False, error=False: messages.append((message, warn, error)), + ) + + info = {"ra": 1.078996153, "dec": 74.3313055} + + corrected = exotic_module.maybe_reinterpret_decimal_ra_hours_from_wcs(["frame.fits"], info) + + assert corrected is True + assert info["ra"] == pytest.approx(16.184942295) + assert any("interpreted decimal target RA as hours" in message and warn for message, warn, _ in messages) + + already_degrees = {"ra": 16.184942295, "dec": 74.3313055} + + corrected_again = exotic_module.maybe_reinterpret_decimal_ra_hours_from_wcs( + ["frame.fits"], + already_degrees, + ) + + assert corrected_again is False + assert already_degrees["ra"] == pytest.approx(16.184942295) + + +def test_filter_pointing_outlier_frames_uses_wcs_when_all_frames_have_wcs(monkeypatch): + frames = [f"frame_{i}.fits" for i in range(6)] + wcs_positions = np.array( + [ + [100.0, 100.0], + [101.0, 100.0], + [99.0, 100.0], + [100.0, 101.0], + [100.0, 99.0], + [0.0, 0.0], + ], + dtype=float, + ) + + monkeypatch.setattr( + exotic_module, + "collect_wcs_frame_center_pointings", + lambda inputfiles: (wcs_positions, np.ones(len(inputfiles), dtype=bool)), + ) + monkeypatch.setattr( + exotic_module, + "collect_transform_frame_pointings", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("transform fallback should not be used")), + ) + + filtered, keep_mask, dropped = exotic_module.filter_pointing_outlier_frames( + frames, + pointing_rejection_sigma=3.0, + ) + + assert filtered.tolist() == frames[:-1] + assert keep_mask.tolist() == [True, True, True, True, True, False] + assert dropped == [frames[-1]] + + +def test_filter_pointing_outlier_frames_falls_back_to_transform_when_wcs_is_incomplete(monkeypatch): + frames = [f"frame_{i}.fits" for i in range(6)] + transform_positions = np.array( + [ + [50.0, 50.0], + [50.5, 49.5], + [49.5, 50.5], + [50.0, 51.0], + [50.0, 49.0], + [10.0, 10.0], + ], + dtype=float, + ) + transform_calls = [] + + monkeypatch.setattr( + exotic_module, + "collect_wcs_frame_center_pointings", + lambda inputfiles: ( + np.full((len(inputfiles), 2), np.nan, dtype=float), + np.array([True, True, True, True, False, False], dtype=bool), + ), + ) + + def fake_collect_transform_frame_pointings(inputfiles, frame_loader=None, return_transforms=False, **kwargs): + transform_calls.append((tuple(inputfiles), frame_loader, return_transforms, kwargs)) + transforms = { + str(file_name): exotic_module.SimilarityTransform(scale=1, rotation=0, translation=[index, 0]) + for index, file_name in enumerate(inputfiles) + } + if return_transforms: + return transform_positions, np.ones(len(inputfiles), dtype=bool), transforms + return transform_positions, np.ones(len(inputfiles), dtype=bool) + + monkeypatch.setattr(exotic_module, "collect_transform_frame_pointings", fake_collect_transform_frame_pointings) + + filtered, keep_mask, dropped, cached_transforms = exotic_module.filter_pointing_outlier_frames( + frames, + pointing_rejection_sigma=3.0, + allow_pixel_alignment_fallback=True, + return_alignment_transforms=True, + ) + + assert len(transform_calls) == 1 + assert transform_calls[0][2] is True + assert filtered.tolist() == frames[:-1] + assert keep_mask.tolist() == [True, True, True, True, True, False] + assert dropped == [frames[-1]] + assert set(cached_transforms) == set(frames[:-1]) + + +def test_reference_frame_rejection_fallback_reports_automatic_removal_and_reprojection(monkeypatch): + messages = [] + + monkeypatch.setattr( + exotic_module, + "log_info", + lambda message, error=False, warn=False: messages.append((message, error, warn)), + ) + + result = exotic_module.reference_frame_rejection_fallback_info( + "frame_0001.fits", + ["frame_0001.fits", "frame_0002.fits", "frame_0003.fits"], + ordered_inputfiles=[ + "frame_0001.fits", + "frame_0002.fits", + "frame_0003.fits", + "frame_0004.fits", + ], + ) + + assert result["leading_rejected_files"] == ["frame_0001.fits", "frame_0002.fits", "frame_0003.fits"] + assert result["next_reference_candidate"] == "frame_0004.fits" + assert any("automatically removing" in message and warn for message, _, warn in messages) + assert any("target RA/Dec" in message and "nextastro_archive" in message and warn for message, _, warn in messages) + assert any( + "Automatically removed leading rejected frame(s)" in message + and "frame_0001.fits, frame_0002.fits, frame_0003.fits" in message + and "Continuing from new reference image frame_0004.fits" in message + and warn + for message, _, warn in messages + ) + + +def test_reference_frame_rejection_fallback_only_reports_consecutive_leading_rejections(monkeypatch): + messages = [] + + monkeypatch.setattr( + exotic_module, + "log_info", + lambda message, error=False, warn=False: messages.append((message, error, warn)), + ) + + result = exotic_module.reference_frame_rejection_fallback_info( + "frame_0001.fits", + ["frame_0001.fits", "frame_0003.fits"], + ordered_inputfiles=[ + "frame_0001.fits", + "frame_0002.fits", + "frame_0003.fits", + "frame_0004.fits", + ], + ) + + assert result["leading_rejected_files"] == ["frame_0001.fits"] + assert result["next_reference_candidate"] == "frame_0002.fits" + assert any( + "Automatically removed leading rejected frame(s)" in message + and "frame_0001.fits" in message + and "Continuing from new reference image frame_0002.fits" in message + and "frame_0003.fits" not in message + and warn + for message, _, warn in messages + ) + + +def test_reference_frame_rejection_fallback_ignores_non_reference_rejections(monkeypatch): + messages = [] + + monkeypatch.setattr( + exotic_module, + "log_info", + lambda message, error=False, warn=False: messages.append((message, error, warn)), + ) + + result = exotic_module.reference_frame_rejection_fallback_info( + "frame_0001.fits", + ["frame_0002.fits", "frame_0003.fits"], + ) + + assert result is None + assert messages == [] + + +def test_reference_fallback_comparison_stars_use_nextastro_archive_image_criteria(): + image = np.zeros((300, 300), dtype=float) + + def add_blob(x_pos, y_pos, value): + image[y_pos - 1:y_pos + 2, x_pos - 1:x_pos + 2] = value * 0.5 + image[y_pos, x_pos] = value + + add_blob(150, 150, 2000.0) # target location, excluded by detected-target match + add_blob(220, 220, 1200.0) + add_blob(80, 80, 900.0) + add_blob(180, 180, 1600.0) # within 50 px of target, excluded + add_blob(25, 25, 5000.0) # outside the central 50% frame, excluded + + comp_stars, candidates = exotic_module.select_reference_fallback_comparison_stars( + image, + image.shape, + target_pixel=[150, 150], + comp_count=2, + ) + + assert comp_stars == [[220.0, 220.0], [80.0, 80.0]] + assert [candidate["flux"] for candidate in candidates] == sorted( + [candidate["flux"] for candidate in candidates], + reverse=True, + ) + + +def test_automatic_optimal_calibration_selector_filters_flux_and_ranks_color(monkeypatch): + image = np.zeros((300, 300), dtype=float) + + def add_blob(x_pos, y_pos, value): + image[y_pos - 1:y_pos + 2, x_pos - 1:x_pos + 2] = value * 0.5 + image[y_pos, x_pos] = value + + add_blob(150, 150, 2000.0) + add_blob(220, 220, 1700.0) + add_blob(80, 80, 1800.0) + add_blob(230, 80, 6000.0) + + ra_wcs = np.tile(np.arange(300, dtype=float), (300, 1)) + dec_wcs = np.tile(np.arange(300, dtype=float)[:, None], (1, 300)) + catalog = {"rows": []} + + def fake_color_match(_catalog, ra, dec, obs_filter, max_separation_arcsec=5.0, **kwargs): + colors = { + (150, 150): (12.0, 11.4), + (220, 220): (13.0, 12.41), + (80, 80): (13.0, 12.0), + (230, 80): (10.0, 9.4), + } + key = (int(round(float(ra))), int(round(float(dec)))) + if key not in colors: + return None + b_mag, v_mag = colors[key] + return { + "catalog_row": {"Bmag": b_mag, "Vmag": v_mag}, + "color": { + "color": b_mag - v_mag, + "label": "B-V", + "first_column": "Bmag", + "second_column": "Vmag", + }, + } + + monkeypatch.setattr(exotic_module, "nextastro_catalog_nearest_color_row", fake_color_match) + monkeypatch.setattr( + exotic_module, + "nextastro_photometry_catalog_match", + lambda catalog_response, ra, dec, obs_filter, **kwargs: ( + { + **fake_color_match(catalog_response, ra, dec, obs_filter), + "mag": 12.0, + "error": 0.01, + "mag_band": "V", + } + if fake_color_match(catalog_response, ra, dec, obs_filter) is not None + else None + ), + ) + + comp_stars, candidates = exotic_module.select_automatic_optimal_calibration_stars( + image, + image.shape, + target_pixel=[150, 150], + ra_wcs=ra_wcs, + dec_wcs=dec_wcs, + obs_filter="V", + field_catalog=catalog, + count=2, + colour_term_metadata={ + "term": 0.2, + "term_error": 0.01, + "term_index": "B-V", + }, + ) + + assert comp_stars[0] == [220.0, 220.0] + assert [candidate["color_delta"] for candidate in candidates] == sorted( + candidate["color_delta"] for candidate in candidates + ) + assert candidates[0]["expected_colour_mismatch_mag"] == pytest.approx( + 0.2 * candidates[0]["color_delta"] + ) + assert candidates[0]["colour_term_uncertainty_mag"] == pytest.approx( + 0.01 * candidates[0]["color_delta"] + ) + + brightest_comp_stars, brightest_candidates = exotic_module.select_automatic_optimal_calibration_stars( + image, + image.shape, + target_pixel=[150, 150], + ra_wcs=ra_wcs, + dec_wcs=dec_wcs, + obs_filter="V", + field_catalog=catalog, + count=2, + brightest_first=True, + saturation_threshold=5000.0, + ) + + assert brightest_comp_stars[0] == [80.0, 80.0] + assert [candidate["flux"] for candidate in brightest_candidates] == sorted( + [candidate["flux"] for candidate in brightest_candidates], + reverse=True, + ) + assert all(0.5 <= candidate["brightness_ratio"] <= 2.0 for candidate in candidates) + + +def test_build_absolute_comp_ensemble_flux_uses_median_normalized_members(): + comp_flux_map = { + "comp1": np.array([100.0, 102.0, 98.0, 100.0, 101.0, 99.0]), + "comp2": np.array([200.0, 204.0, 196.0, 200.0, 202.0, 198.0]), + "comp3": np.array([np.nan, np.nan, np.nan, np.nan, np.nan, np.nan]), + } + + ensemble_flux, member_keys = exotic_module.build_absolute_comp_ensemble_flux( + comp_flux_map, + ["comp1", "comp2", "comp3"], + ) + + assert member_keys == ["comp1", "comp2"] + assert np.nanmedian(ensemble_flux) == pytest.approx(150.0) + assert ensemble_flux[1] / np.nanmedian(ensemble_flux) == pytest.approx(1.02) diff --git a/tests/test_elca_baseline.py b/tests/test_elca_baseline.py new file mode 100644 index 00000000..53966fe9 --- /dev/null +++ b/tests/test_elca_baseline.py @@ -0,0 +1,2386 @@ +import importlib +import sys +import types + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from matplotlib.axes import Axes +import numpy as np +import pytest + + +def load_elca_with_stubs(monkeypatch, tmp_path): + root = tmp_path / "stubdeps" + package_dir = root / "pylightcurve" + model_dir = package_dir / "models" + model_dir.mkdir(parents=True) + + (package_dir / "__init__.py").write_text("", encoding="utf-8") + (model_dir / "__init__.py").write_text("", encoding="utf-8") + (model_dir / "exoplanet_lc.py").write_text( + "import numpy as np\n" + "def transit(ld, rprs, per, ars, ecc, inc, omega, tmid, times, method=None, precision=None):\n" + " times = np.asarray(times, dtype=float)\n" + " return 1.0 - (rprs ** 2) * np.exp(-0.5 * ((times - tmid) / 0.01) ** 2)\n", + encoding="utf-8", + ) + + monkeypatch.syspath_prepend(str(root)) + + for name in list(sys.modules): + if name == "exotic.api.elca" or name.startswith("pylightcurve"): + sys.modules.pop(name, None) + + fake_ultranest = types.ModuleType("ultranest") + fake_ultranest.ReactiveNestedSampler = type("ReactiveNestedSampler", (), {}) + fake_plotting = types.ModuleType("plotting") + fake_plotting.corner = lambda *args, **kwargs: None + fake_ultranest_utils = types.ModuleType("ultranest_utils") + fake_ultranest_utils.run_reactive_sampler = lambda *args, **kwargs: None + + monkeypatch.setitem(sys.modules, "ultranest", fake_ultranest) + monkeypatch.setitem(sys.modules, "plotting", fake_plotting) + monkeypatch.setitem(sys.modules, "exotic.api.plotting", fake_plotting) + monkeypatch.setitem(sys.modules, "ultranest_utils", fake_ultranest_utils) + monkeypatch.setitem(sys.modules, "exotic.api.ultranest_utils", fake_ultranest_utils) + + import exotic.api.elca as elca + + return importlib.reload(elca) + + +def make_prior(): + return { + "rprs": 0.1, + "ars": 12.0, + "per": 3.0, + "inc": 89.0, + "u0": 0.0, + "u1": 0.0, + "u2": 0.0, + "u3": 0.0, + "ecc": 0.0, + "omega": 90.0, + "tmid": 0.0, + "a0": 1.0, + "a2": 0.0, + } + + +def make_expanded_prior_warmstart_source( + prior, + time, + data, + dataerr, + airmass, + sample_points, +): + sample_points = np.asarray(sample_points, dtype=float) + sample_count = sample_points.shape[0] + return types.SimpleNamespace( + ns_type="ultranest", + bounds={ + "rprs": [0.08, 0.12], + "tmid": [-0.005, 0.005], + }, + sampled_keys=["rprs", "tmid"], + prior=prior.copy(), + time=np.asarray(time, dtype=float), + data=np.asarray(data, dtype=float), + dataerr=np.asarray(dataerr, dtype=float), + airmass=np.asarray(airmass, dtype=float), + exposure_times_days=None, + baseline_fit_mask=None, + duration_prior=None, + fixed_flux_baseline=False, + use_impactparameter_rather_than_inclination_to_fit=False, + results={ + "weighted_samples": { + "points": sample_points, + "weights": np.full(sample_count, 1.0 / sample_count), + "logl": np.linspace(-5.0, -1.0, sample_count), + }, + }, + ) + + +def make_dummy_nested_result(sample_points, auxiliary=False): + sample_points = np.asarray(sample_points, dtype=float) + if auxiliary: + result_points = np.column_stack([ + sample_points, + np.zeros(sample_points.shape[0], dtype=float), + ]) + else: + result_points = sample_points + parameter_count = result_points.shape[1] + maximum_likelihood = np.zeros(parameter_count, dtype=float) + maximum_likelihood[0] = 0.1 + return { + "maximum_likelihood": {"point": maximum_likelihood}, + "posterior": { + "stdev": np.full(parameter_count, 0.001), + "errlo": np.full(parameter_count, -0.001), + "errup": np.full(parameter_count, 0.001), + }, + "weighted_samples": { + "points": result_points, + "weights": np.full(result_points.shape[0], 1.0 / result_points.shape[0]), + "logl": np.linspace(-5.0, -1.0, result_points.shape[0]), + }, + "samples": result_points.copy(), + } + + +def test_lc_fitter_recovers_explicit_a0_baseline(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 301) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = 0.98 * elca.transit(time, prior) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005], "a0": [0.95, 1.05]}, + mode="lm", + verbose=False, + ) + + oot_mask = np.abs(fit.time - fit.parameters["tmid"]) > 0.02 + assert fit.parameters["a0"] == pytest.approx(0.98, abs=1e-4) + assert fit.parameters["a1"] == pytest.approx(0.98, abs=1e-4) + assert np.median(fit.detrended[oot_mask]) == pytest.approx(1.0, abs=5e-4) + + +def test_lc_fitter_explicit_a0_tracks_mean_airmass_normalization(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + prior["a2"] = -0.35 + time = np.linspace(-0.03, 0.03, 301) + airmass = np.linspace(1.15, 1.85, len(time)) + dataerr = np.full_like(time, 1e-3) + true_a0 = 0.985 + data = true_a0 * elca.airmass_trend(prior["a2"], airmass) * elca.transit(time, prior) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005], "a0": [0.95, 1.05]}, + mode="lm", + verbose=False, + ) + + assert fit.airmass_reference == pytest.approx(np.mean(airmass)) + assert fit.parameters["a0"] == pytest.approx(true_a0, abs=1e-4) + assert fit.parameters["a1"] == pytest.approx(true_a0, abs=1e-4) + + +def test_lc_fitter_auto_solves_baseline_when_a0_is_not_free(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 301) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = 1.03 * elca.transit(time, prior) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005]}, + mode="lm", + verbose=False, + ) + + oot_mask = np.abs(fit.time - fit.parameters["tmid"]) > 0.02 + assert fit.parameters["a0"] == pytest.approx(1.03, abs=1e-4) + assert fit.parameters["a1"] == pytest.approx(1.03, abs=1e-4) + assert np.median(fit.detrended[oot_mask]) == pytest.approx(1.0, abs=5e-4) + + +def test_lc_fitter_falls_back_to_lm_after_nested_linalg_error(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + calls = [] + + def fake_fit_nested(self): + calls.append("ns") + raise np.linalg.LinAlgError("Singular matrix") + + def fake_fit_LM(self): + calls.append("lm") + self.parameters = self.prior.copy() + self.errors = {} + self.quantiles = {} + self.sampled_keys = [] + self.sample_bounds = {} + self.sample_parameters = {} + self.sample_errors = {} + self.sample_quantiles = {} + + monkeypatch.setattr(elca.lc_fitter, "fit_nested", fake_fit_nested) + monkeypatch.setattr(elca.lc_fitter, "fit_LM", fake_fit_LM) + + prior = make_prior() + time = np.linspace(-0.03, 0.03, 31) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = elca.transit(time, prior) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005]}, + mode="ns", + verbose=False, + ) + + assert calls == ["ns", "lm"] + assert fit.mode == "lm" + assert fit.ns_type == "lm" + assert fit.nested_fit_fallback is True + assert fit.nested_fit_failure_reason == "LinAlgError: Singular matrix" + + +def test_lc_fitter_auto_solves_mean_airmass_normalization(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + prior["a2"] = 0.22 + time = np.linspace(-0.03, 0.03, 301) + airmass = np.linspace(1.05, 1.75, len(time)) + dataerr = np.full_like(time, 1e-3) + true_a0 = 1.018 + data = true_a0 * elca.airmass_trend(prior["a2"], airmass) * elca.transit(time, prior) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005]}, + mode="lm", + verbose=False, + ) + + assert fit.parameters["a0"] == pytest.approx(true_a0, abs=1e-4) + assert fit.parameters["a1"] == pytest.approx(true_a0, abs=1e-4) + + +def test_create_fit_variables_solves_baseline_from_out_of_transit_mask(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 301) + dataerr = np.full_like(time, 1e-3) + airmass = np.zeros_like(time) + transit_model = elca.transit(time, prior) + data = 1.02 * transit_model + in_transit = np.abs(time - prior["tmid"]) < 0.012 + data[in_transit] *= 0.90 + + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.time = time + fit.data = data + fit.dataerr = dataerr + fit.airmass = airmass + fit.airmass_reference = elca.get_airmass_reference(airmass) + fit.prior = prior.copy() + fit.bounds = {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005]} + fit.mode = "ns" + fit.parameters = prior.copy() + fit.errors = {"rprs": 0.0, "tmid": 0.0, "a2": 0.0} + fit.quantiles = {} + fit.baseline_fit_mask = ~in_transit + fit.fixed_parameter_errors = {} + + fit.create_fit_variables() + + assert fit.parameters["a0"] == pytest.approx(1.02, abs=1e-5) + assert fit.parameters["a1"] == pytest.approx(1.02, abs=1e-5) + + +def test_lc_fitter_rejects_redundant_a0_and_a1_bounds(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + prior["a1"] = 1.0 + time = np.linspace(-0.03, 0.03, 31) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = elca.transit(time, prior) + + with pytest.raises(ValueError, match="Use only one of 'a0' or 'a1'"): + elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior, + {"a0": [0.95, 1.05], "a1": [0.95, 1.05]}, + mode="lm", + verbose=False, + ) + + +def test_create_fit_variables_preserves_explicit_baseline_in_nested_mode(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + sampled = make_prior() + sampled["rprs"] = 0.09 + sampled["a0"] = 0.98 + sampled["a1"] = 0.98 + + truth = make_prior() + truth["rprs"] = 0.12 + + fit.time = np.linspace(-0.03, 0.03, 301) + fit.data = elca.transit(fit.time, truth) + fit.dataerr = np.full_like(fit.time, 1e-3) + fit.airmass = np.zeros_like(fit.time) + fit.prior = sampled.copy() + fit.bounds = {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005], "a0": [0.95, 1.05]} + fit.mode = "ns" + fit.parameters = sampled.copy() + fit.errors = {"rprs": 1e-3, "tmid": 1e-4, "a0": 2e-3} + + fit.create_fit_variables() + + assert fit.parameters["a0"] == pytest.approx(0.98, abs=1e-9) + assert fit.parameters["a1"] == pytest.approx(0.98, abs=1e-9) + + +def test_create_fit_variables_respects_plot_time_range(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + prior = make_prior() + fit.time = np.array([-0.015, 0.010], dtype=float) + fit.data = 0.99 * elca.transit(fit.time, prior) + fit.dataerr = np.full_like(fit.time, 1e-3) + fit.airmass = np.zeros_like(fit.time) + fit.prior = prior.copy() + fit.bounds = {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005], "a0": [0.95, 1.05]} + fit.mode = "ns" + fit.parameters = prior.copy() + fit.errors = {"rprs": 1e-3, "tmid": 1e-4, "a0": 2e-3} + fit.plot_time_range = (-0.12, 0.18) + + fit.create_fit_variables() + + assert fit.time_upsample[0] == pytest.approx(-0.12, abs=1e-12) + assert fit.time_upsample[-1] == pytest.approx(0.18, abs=1e-12) + assert fit.phase_upsample[0] == pytest.approx(-0.04, abs=1e-12) + assert fit.phase_upsample[-1] == pytest.approx(0.06, abs=1e-12) + + +def test_plot_bestfit_uses_full_plot_time_range_for_phase_xlim(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.015, 0.010, 51) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = 0.99 * elca.transit(time, prior) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005], "a0": [0.95, 1.05]}, + mode="lm", + verbose=False, + ) + fit.plot_time_range = (-0.12, 0.18) + fit._update_plot_geometry() + + fig, axes = fit.plot_bestfit() + + assert axes[0].get_xlim() == pytest.approx((-0.04, 0.06), abs=1e-6) + assert axes[1].get_xlim() == pytest.approx((-0.04, 0.06), abs=1e-6) + plt.close(fig) + + +def test_plot_bestfit_can_hide_flux_baseline_label(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.015, 0.010, 51) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = 0.99 * elca.transit(time, prior) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005], "a0": [0.95, 1.05]}, + mode="lm", + verbose=False, + ) + + fig, axes = fit.plot_bestfit(show_flux_baseline_label=False) + legend_text = "\n".join(text.get_text() for text in axes[0].get_legend().get_texts()) + + assert "$a_0$" not in legend_text + plt.close(fig) + + +def test_format_value_error_for_plot_preserves_two_sigfig_uncertainty_places(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + + assert elca.format_value_error_for_plot(0.027, 0.05) == ("0.027", "0.050") + assert elca.format_value_error_for_plot(2461209.81, 0.087) == ("2461209.810", "0.087") + assert elca.format_value_error_for_plot(89.3511, 2.16) == ("89.4", "2.2") + + +def test_plot_bestfit_marks_prior_rprs_fallback_uncertainty(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.015, 0.010, 51) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = 0.99 * elca.transit(time, prior) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"tmid": [-0.005, 0.005], "a0": [0.95, 1.05]}, + mode="lm", + verbose=False, + fixed_parameter_errors={"rprs": 0.02}, + ) + fit.rprs_prior_fallback_applied = True + fit.empirical_transit_uncertainty = { + "available": True, + "combined_rprs_uncertainty": 0.02, + } + + fig, axes = fit.plot_bestfit(show_flux_baseline_label=False) + legend_text = "\n".join(text.get_text() for text in axes[0].get_legend().get_texts()) + + assert "(Prior)" in legend_text + assert "0.0100" in legend_text + assert "0.0040" in legend_text + plt.close(fig) + + +def test_plot_bestfit_can_draw_transit_model_uncertainty_band(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.015, 0.010, 51) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = 0.99 * elca.transit(time, prior) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005], "a0": [0.95, 1.05]}, + mode="lm", + verbose=False, + ) + fit.errors["rprs"] = 0.01 + fit.errors["tmid"] = 0.001 + envelope = fit.transit_model_uncertainty(fit.time_upsample) + + fig, axes = fit.plot_bestfit(show_model_uncertainty=True) + labels = [artist.get_label() for artist in axes[0].collections] + legend_text = "\n".join(text.get_text() for text in axes[0].get_legend().get_texts()) + uncertainty_line_count = sum(1 for line in axes[0].lines if line.get_linestyle() == "--") + + assert envelope is not None + assert np.nanmax(envelope[1] - envelope[0]) > 0 + assert "_nolegend_" in labels + assert r'1-$\sigma$ model uncertainty' not in legend_text + assert uncertainty_line_count >= 2 + plt.close(fig) + + +def test_plot_bestfit_draws_unbinned_points_black_with_grey_errorbars(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.015, 0.010, 51) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = 0.99 * elca.transit(time, prior) + captured_errorbars = [] + + original_errorbar = Axes.errorbar + + def spy_errorbar(self, *args, **kwargs): + captured_errorbars.append(kwargs.copy()) + return original_errorbar(self, *args, **kwargs) + + monkeypatch.setattr(Axes, "errorbar", spy_errorbar) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005], "a0": [0.95, 1.05]}, + mode="lm", + verbose=False, + ) + + fig, _ = fit.plot_bestfit() + + assert captured_errorbars[0]["color"] == "black" + assert captured_errorbars[0]["ecolor"] == "0.72" + assert captured_errorbars[0]["alpha"] == 1.0 + plt.close(fig) + + +def test_transit_model_uncertainty_includes_baseline_terms(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 51) + + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.time = time + fit.data = elca.transit(time, prior) + fit.dataerr = np.full_like(time, 1e-3) + fit.airmass = np.linspace(1.0, 1.5, time.size) + fit.airmass_reference = elca.get_airmass_reference(fit.airmass) + fit.prior = prior.copy() + fit.bounds = {} + fit.mode = "ns" + fit.parameters = prior.copy() + fit.parameters["a0"] = 1.0 + fit.parameters["a1"] = 1.0 + fit.parameters["a2"] = 0.1 + fit.errors = {"a0": 0.01, "a1": 0.01, "a2": 0.05} + fit.quantiles = {} + fit.results = None + + envelope = fit.transit_model_uncertainty(time) + + assert envelope is not None + assert np.nanmax(envelope[1] - envelope[0]) > 0 + + +def test_baseline_model_uncertainty_is_centered_on_unity_and_includes_a2(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 51) + + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.time = time + fit.airmass = np.linspace(1.0, 2.0, time.size) + fit.airmass_reference = elca.get_airmass_reference(fit.airmass) + fit.parameters = prior.copy() + fit.parameters["a0"] = 1.0 + fit.parameters["a1"] = 1.0 + fit.parameters["a2"] = 0.1 + fit.errors = {"a0": 0.01, "a1": 0.01, "a2": 0.05} + + lower, upper = fit.baseline_model_uncertainty(time) + width = upper - lower + + np.testing.assert_allclose(0.5 * (lower + upper), np.ones_like(time), atol=1e-12) + assert np.nanmin(lower) < 1.0 + assert np.nanmax(upper) > 1.0 + assert width[0] > width[len(width) // 2] + + +def test_baseline_model_uncertainty_does_not_double_count_analytic_a0(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 51) + airmass = np.linspace(1.0, 2.0, time.size) + + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.time = time + fit.data = elca.transit(time, prior) + fit.dataerr = np.full_like(time, 1e-3) + fit.airmass = airmass + fit.airmass_reference = elca.get_airmass_reference(fit.airmass) + fit.prior = prior.copy() + fit.bounds = {"a2": [-1.0, 1.0]} + fit.fixed_flux_baseline = False + fit.mode = "ns" + fit.parameters = prior.copy() + fit.parameters["a0"] = 1.0 + fit.parameters["a1"] = 1.0 + fit.parameters["a2"] = 0.0 + fit.errors = {"a0": 0.5, "a1": 0.5, "a2": 0.02} + fit.results = None + + lower, upper = fit.baseline_model_uncertainty(time) + half_width = np.nanmax(np.maximum(1.0 - lower, upper - 1.0)) + + assert half_width < 0.03 + + +def test_baseline_model_uncertainty_includes_empirical_flux_floor(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 51) + + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.time = time + fit.airmass = np.linspace(1.0, 2.0, time.size) + fit.airmass_reference = elca.get_airmass_reference(fit.airmass) + fit.parameters = prior.copy() + fit.parameters["a0"] = 1.0 + fit.parameters["a1"] = 1.0 + fit.parameters["a2"] = 0.0 + fit.errors = {"a0": 0.001, "a1": 0.001, "a2": 0.001} + fit.results = None + fit.empirical_transit_uncertainty = { + "available": True, + "baseline_red_noise_uncertainty_fraction": 0.02, + } + + lower, upper = fit.baseline_model_uncertainty(time) + half_width = np.nanmax(np.maximum(1.0 - lower, upper - 1.0)) + + assert half_width >= 0.02 + + +def test_baseline_model_uncertainty_prefers_posterior_samples(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 51) + sample_count = 41 + a0_samples = 1.0 + np.linspace(-0.004, 0.004, sample_count) + a2_samples = np.linspace(-0.02, 0.02, sample_count) + + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.time = time + fit.data = elca.transit(time, prior) + fit.dataerr = np.full_like(time, 1e-3) + fit.airmass = np.linspace(1.0, 2.0, time.size) + fit.airmass_reference = elca.get_airmass_reference(fit.airmass) + fit.prior = prior.copy() + fit.bounds = {"a0": [0.5, 1.5], "a2": [-1.0, 1.0]} + fit.sampled_keys = ["a0", "a2"] + fit.mode = "ns" + fit.ns_type = "ultranest" + fit.parameters = prior.copy() + fit.parameters["a0"] = 1.0 + fit.parameters["a1"] = 1.0 + fit.parameters["a2"] = 0.0 + fit.errors = {"a0": 0.5, "a1": 0.5, "a2": 0.5} + fit.results = { + "weighted_samples": { + "points": np.column_stack([a0_samples, a2_samples]), + "logl": np.zeros(sample_count, dtype=float), + "weights": np.ones(sample_count, dtype=float), + } + } + + lower, upper = fit.baseline_model_uncertainty(time) + half_width = np.nanmax(np.maximum(1.0 - lower, upper - 1.0)) + + assert half_width < 0.02 + + +def test_plot_bestfit_can_draw_baseline_uncertainty_band(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 51) + airmass = np.linspace(1.0, 2.0, time.size) + dataerr = np.full_like(time, 1e-3) + data = 0.99 * elca.transit(time, prior) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "tmid": [-0.005, 0.005], "a0": [0.95, 1.05]}, + mode="lm", + verbose=False, + ) + fit.parameters["a2"] = 0.1 + fit.errors["a0"] = 0.01 + fit.errors["a2"] = 0.05 + + fig, axes = fit.plot_bestfit(show_baseline_uncertainty=True) + labels = [artist.get_label() for artist in axes[0].collections] + legend_text = "\n".join(text.get_text() for text in axes[0].get_legend().get_texts()) + baseline_line_count = sum( + 1 + for line in axes[0].lines + if line.get_linestyle() == "--" and line.get_color() == "gold" + ) + + assert "_nolegend_" in labels + assert r'$a_0/a_2$ 1-$\sigma$ baseline uncertainty' not in legend_text + assert baseline_line_count == 2 + plt.close(fig) + + +def test_posterior_model_uncertainty_recenters_on_best_fit_model(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 51) + + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.time = time + fit.data = elca.transit(time, prior) + fit.dataerr = np.full_like(time, 1e-3) + fit.airmass = np.zeros_like(time) + fit.airmass_reference = elca.get_airmass_reference(fit.airmass) + fit.prior = prior.copy() + fit.bounds = {"a0": [0.99, 1.03]} + fit.sampled_keys = ["a0"] + fit.sample_bounds = {"a0": [0.99, 1.03]} + fit.mode = "ns" + fit.ns_type = "ultranest" + fit.parameters = prior.copy() + fit.parameters["a0"] = 1.0 + fit.parameters["a1"] = 1.0 + fit.errors = {"a0": 0.002} + fit.quantiles = {} + fit.results = { + "weighted_samples": { + "points": np.linspace(1.008, 1.012, 41)[:, None], + "logl": np.zeros(41, dtype=float), + "weights": np.ones(41, dtype=float), + } + } + + lower, upper = fit.transit_model_uncertainty(time) + center = 0.5 * (lower + upper) + + np.testing.assert_allclose(center, elca.transit(time, fit.parameters), atol=5e-5) + assert np.nanmedian(center[:3]) < 1.001 + + +def test_glc_plot_bestfit_median_limits_use_full_phase_span(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + phase = np.array([0.01, 0.02], dtype=float) + phase_upsample = np.linspace(-0.04, 0.06, 100) + residuals = np.array([1e-4, -1e-4], dtype=float) + times = prior["tmid"] + phase * prior["per"] + + fit = elca.glc_fitter.__new__(elca.glc_fitter) + fit.parameters = prior.copy() + fit.errors = {"rprs": 1e-3, "tmid": 1e-4} + fit.lc_data = [{ + "time": times, + "flux": np.ones_like(times), + "detrend": np.ones_like(times), + "ferr": np.full_like(times, 1e-3), + "residuals": residuals, + "phase": phase, + "phase_upsample": phase_upsample, + "time_upsample": prior["tmid"] + phase_upsample * prior["per"], + "transit_upsample": np.ones_like(phase_upsample), + "priors": prior.copy(), + "errors": {"rprs": 1e-3, "tmid": 1e-4}, + "name": "dataset", + }] + + fig, axes = fit.plot_bestfit(phase_limits="median") + + assert axes[0].get_xlim() == pytest.approx((-0.04, 0.06), abs=1e-6) + assert axes[1].get_xlim() == pytest.approx((-0.04, 0.06), abs=1e-6) + plt.close(fig) + + +def test_plot_triangle_clips_ranges_to_parameter_bounds(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + captured = {} + + def fake_corner(*args, **kwargs): + captured["points"] = args[0] + captured["labels"] = kwargs["labels"] + captured["range"] = kwargs["range"] + captured["titles"] = kwargs["titles"] + captured["title_kwargs"] = kwargs["title_kwargs"] + captured["label_kwargs"] = kwargs["label_kwargs"] + return "figure" + + monkeypatch.setattr(elca, "corner", fake_corner) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.125], + "inc": [84.0, 90.0], + "a0": [0.95, 1.05], + } + fit.prior = make_prior() + fit.quantiles = {"rprs": [], "inc": [], "a0": []} + fit.parameters = {"rprs": 0.10, "inc": 88.42, "a0": 0.94962} + fit.errors = {"rprs": 0.01, "inc": 0.75, "a0": 0.00394} + + points = np.array( + [ + [0.099, 88.30, 0.9501], + [0.101, 88.55, 0.9502], + [0.102, 88.10, 0.9515], + [0.098, 88.70, 0.9520], + [0.100, 88.40, 0.9508], + ] + ) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.array([-5.0, -4.0, -4.5, -5.5, -4.2]), + }, + "samples": points.copy(), + } + + fig = fit.plot_triangle() + + assert fig == "figure" + assert captured["labels"][1] == r"$\Delta i$" + assert captured["range"][0][0] == pytest.approx(0.0) + assert captured["range"][0][1] == pytest.approx(0.125) + expected_inc_distance_limit = np.max(np.abs(np.array([84.0, 90.0]) - fit.parameters["inc"])) + assert captured["range"][1][0] == pytest.approx(-expected_inc_distance_limit) + assert captured["range"][1][1] == pytest.approx(expected_inc_distance_limit) + assert captured["range"][2][0] == pytest.approx(0.95) + assert captured["range"][2][1] == pytest.approx(1.05) + assert captured["points"].shape == (10, 3) + expected_inc_distance = np.abs(points[:, 1] - fit.parameters["inc"]) + np.testing.assert_allclose(captured["points"][:5, 1], expected_inc_distance) + np.testing.assert_allclose(captured["points"][5:, 1], -expected_inc_distance) + assert captured["titles"][1].startswith("b=") + assert "\ni=" in captured["titles"][1] + assert captured["title_kwargs"]["loc"] == "left" + assert captured["label_kwargs"]["labelpad"] == 10 + + +def test_internal_impact_parameter_transform_round_trips_inclination(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.mode = "ns" + fit.use_impactparameter_rather_than_inclination_to_fit = True + fit.prior = make_prior() + fit.bounds = { + "rprs": [0.08, 0.12], + "inc": [87.0, 90.0], + "tmid": [-0.005, 0.005], + } + + sample_point = fit._sample_point_from_unit_cube(np.array([0.25, 0.4, 0.75])) + physical = fit._physical_values_from_sample_point(sample_point) + expected_rprs = 0.08 + 0.25 * (0.12 - 0.08) + expected_b = 0.4 * (1.0 + expected_rprs) + expected_inc = float(elca.inclination_from_impact_parameter( + {**fit.prior, "rprs": expected_rprs}, + expected_b, + )) + + assert fit._get_sampled_keys() == ["rprs", "b", "tmid"] + assert sample_point[1] == pytest.approx(expected_b) + assert physical["inc"] == pytest.approx(expected_inc) + assert physical["b"] == pytest.approx(sample_point[1]) + + +def test_internal_impact_parameter_samples_grazing_range_beyond_one(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.mode = "ns" + fit.use_impactparameter_rather_than_inclination_to_fit = True + fit.prior = make_prior() + fit.bounds = { + "rprs": [0.08, 0.12], + "inc": [89.8, 90.0], + "tmid": [-0.005, 0.005], + } + + sample_point = fit._sample_point_from_unit_cube(np.array([1.0, 0.99, 0.5])) + physical = fit._physical_values_from_sample_point(sample_point) + + assert sample_point[0] == pytest.approx(0.12) + assert sample_point[1] == pytest.approx(0.99 * 1.12) + assert sample_point[1] > 1.0 + assert physical["inc"] < 90.0 + assert fit._get_sample_bounds()["b"] == pytest.approx([0.0, 1.12]) + + +def test_internal_impact_parameter_transform_does_not_deepcopy_prior(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.mode = "ns" + fit.use_impactparameter_rather_than_inclination_to_fit = True + fit.prior = make_prior() + fit.bounds = { + "rprs": [0.08, 0.12], + "ars": [10.0, 14.0], + "inc": [87.0, 90.0], + "tmid": [-0.005, 0.005], + } + + def fail_deepcopy(value, memo=None): + raise AssertionError("sampling transforms should not deepcopy parameter dictionaries") + + monkeypatch.setattr(elca.copy, "deepcopy", fail_deepcopy) + + sample_point = fit._sample_point_from_unit_cube(np.array([0.25, 0.5, 0.4, 0.75])) + unit_points = np.array([ + [0.25, 0.5, 0.4, 0.75], + [1.0, 0.25, 0.9, 0.5], + ]) + sample_points = fit._sample_point_from_unit_cube(unit_points) + physical = fit._physical_values_from_sample_point(sample_point) + sample_bounds = fit._get_sample_bounds() + + assert sample_point[0] == pytest.approx(0.09) + np.testing.assert_allclose( + sample_points, + np.vstack([fit._sample_point_from_unit_cube(row) for row in unit_points]), + ) + assert physical["b"] == pytest.approx(sample_point[2]) + assert "b" in sample_bounds + + +def test_unit_cube_transform_vectorizes_simple_bounds(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.mode = "ns" + fit.use_impactparameter_rather_than_inclination_to_fit = False + fit.prior = make_prior() + fit.bounds = { + "rprs": [0.08, 0.12], + "inc": [87.0, 90.0], + "tmid": [-0.005, 0.005], + } + + unit_points = np.array([ + [0.0, 0.5, 1.0], + [1.0, 0.25, 0.0], + ]) + + np.testing.assert_allclose( + fit._sample_point_from_unit_cube(unit_points), + np.array([ + [0.08, 88.5, 0.005], + [0.12, 87.75, -0.005], + ]), + ) + + +def test_unit_cube_inverse_maps_expanded_prior_samples_back_to_unit_cube(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.mode = "ns" + fit.use_impactparameter_rather_than_inclination_to_fit = False + fit.prior = make_prior() + fit.bounds = { + "rprs": [0.05, 0.15], + "tmid": [-0.01, 0.01], + } + + unit_points = np.array([ + [0.30, 0.25], + [0.70, 0.75], + ]) + sample_points = fit._sample_point_from_unit_cube(unit_points) + + np.testing.assert_allclose( + fit._unit_cube_from_sample_points(sample_points), + unit_points, + ) + + +def test_expanded_prior_warmstart_uses_corrected_guarded_auxiliary_problem(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 81) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = elca.transit(time, prior) + source_points = np.column_stack([ + np.linspace(0.085, 0.115, 64), + np.linspace(-0.004, 0.004, 64), + ]) + source = make_expanded_prior_warmstart_source( + prior, + time, + data, + dataerr, + airmass, + source_points, + ) + captured = {} + + class DummySampler: + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + captured["sampler"] = self + + def fake_run_reactive_sampler(sampler, *args, **kwargs): + unit_values = np.array([ + [0.4, 0.6, 0.25], + [0.4, 0.6, 0.75], + ]) + transformed = np.asarray(sampler.args[2](unit_values), dtype=float) + captured["transformed"] = transformed + captured["physical_loglike"] = np.asarray( + [ + sampler.args[1](np.append(row[:2], 0.0)) + for row in transformed + ], + dtype=float, + ) + captured["corrected_loglike"] = np.asarray( + sampler.args[1](transformed), + dtype=float, + ) + return make_dummy_nested_result(source_points, auxiliary=True) + + monkeypatch.setattr(elca, "ReactiveNestedSampler", DummySampler) + monkeypatch.setattr(elca, "run_reactive_sampler", fake_run_reactive_sampler) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + { + "rprs": [0.05, 0.15], + "tmid": [-0.005, 0.005], + }, + mode="ns", + verbose=False, + use_impactparameter_rather_than_inclination_to_fit=False, + ultranest_warmstart_source=source, + ) + + assert captured["sampler"].args[0] == ["rprs", "tmid", "aux_logweight"] + np.testing.assert_allclose(captured["transformed"][0, :2], [0.09, 0.001]) + assert np.all(np.isfinite(captured["transformed"])) + assert captured["transformed"][1, 0] > captured["transformed"][0, 0] + np.testing.assert_allclose( + captured["corrected_loglike"] - captured["physical_loglike"], + captured["transformed"][:, 2], + ) + assert fit.ultranest_expanded_prior_warmstart_attempted is True + assert fit.ultranest_expanded_prior_warmstart_applied is True + assert fit.ultranest_expanded_prior_warmstart_expanded_keys == ["rprs"] + assert fit.ultranest_expanded_prior_warmstart_source_sample_count == 64 + assert fit._get_triangle_plot_samples()[0].shape[1] == 2 + + +def test_expanded_prior_warmstart_failure_falls_back_to_clean_sampler(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 81) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = elca.transit(time, prior) + source_points = np.column_stack([ + np.linspace(0.085, 0.115, 64), + np.linspace(-0.004, 0.004, 64), + ]) + source = make_expanded_prior_warmstart_source( + prior, + time, + data, + dataerr, + airmass, + source_points, + ) + samplers = [] + + class DummySampler: + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + samplers.append(self) + + def fake_run_reactive_sampler(sampler, *args, **kwargs): + if len(sampler.args[0]) == 3: + raise RuntimeError("synthetic corrected-warmstart failure") + return make_dummy_nested_result(source_points, auxiliary=False) + + monkeypatch.setattr(elca, "ReactiveNestedSampler", DummySampler) + monkeypatch.setattr(elca, "run_reactive_sampler", fake_run_reactive_sampler) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + { + "rprs": [0.05, 0.15], + "tmid": [-0.005, 0.005], + }, + mode="ns", + verbose=False, + use_impactparameter_rather_than_inclination_to_fit=False, + ultranest_warmstart_source=source, + ) + + assert [sampler.args[0] for sampler in samplers] == [ + ["rprs", "tmid", "aux_logweight"], + ["rprs", "tmid"], + ] + assert fit.ultranest_expanded_prior_warmstart_attempted is True + assert fit.ultranest_expanded_prior_warmstart_applied is False + assert "reran from the full expanded prior" in fit.ultranest_expanded_prior_warmstart_note + assert fit.parameters["rprs"] == pytest.approx(0.1) + + +def test_expanded_prior_warmstart_restores_physical_likelihood_for_best_fit(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + results = { + "maximum_likelihood": { + "point": np.array([0.09, 0.0, 5.0]), + "logl": 5.0, + }, + "weighted_samples": { + "points": np.array([ + [0.09, 0.0, 5.0], + [0.11, 0.0, -5.0], + ]), + "logl": np.array([5.0, -5.0]), + }, + } + + def physical_loglike(points): + points = np.asarray(points, dtype=float) + return -((points[:, 0] - 0.11) / 0.01) ** 2 + + elca.lc_fitter._restore_expanded_prior_physical_likelihoods( + results, + physical_loglike, + 2, + ) + + np.testing.assert_allclose( + results["weighted_samples"]["auxiliary_logl"], + np.array([5.0, -5.0]), + ) + np.testing.assert_allclose( + results["weighted_samples"]["logl"], + np.array([-4.0, 0.0]), + ) + assert results["weighted_samples"]["points"].shape == (2, 2) + assert results["weighted_samples"]["auxiliary_points"].shape == (2, 1) + assert results["maximum_likelihood"]["point"].shape == (2,) + assert results["maximum_likelihood"]["point"][0] == pytest.approx(0.11) + assert results["maximum_likelihood"]["logl"] == pytest.approx(0.0) + + +def test_nested_fit_can_keep_inclination_parameterization_when_requested(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.mode = "ns" + fit.use_impactparameter_rather_than_inclination_to_fit = False + fit.prior = make_prior() + fit.bounds = { + "rprs": [0.08, 0.12], + "inc": [87.0, 90.0], + "tmid": [-0.005, 0.005], + } + + assert fit._get_sampled_keys() == ["rprs", "inc", "tmid"] + + +def test_nested_fit_reports_inclination_from_internal_impact_parameter(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 101) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = elca.transit(time, prior) + data[0] += 5e-5 + + class DummySampler: + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + b_ml = float(elca.impact_parameter_from_inclination(prior, 88.8)) + sample_points = np.array( + [ + [0.100, b_ml - 0.02, 0.0000], + [0.101, b_ml - 0.01, 0.0002], + [0.099, b_ml + 0.01, -0.0001], + [0.100, b_ml + 0.02, 0.0001], + ] + ) + + monkeypatch.setattr(elca, "ReactiveNestedSampler", DummySampler) + monkeypatch.setattr( + elca, + "run_reactive_sampler", + lambda *args, **kwargs: { + "maximum_likelihood": {"point": np.array([0.100, b_ml, 0.0])}, + "posterior": { + "stdev": np.array([0.005, 0.02, 0.0005]), + "errlo": np.array([-0.005, -0.02, -0.0005]), + "errup": np.array([0.005, 0.02, 0.0005]), + }, + "weighted_samples": { + "points": sample_points, + "logl": np.array([-4.0, -3.0, -3.2, -3.8]), + }, + "samples": sample_points.copy(), + }, + ) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "inc": [87.0, 90.0], "tmid": [-0.005, 0.005]}, + mode="ns", + verbose=False, + ) + + assert fit.sampled_keys == ["rprs", "b", "tmid"] + assert fit.sample_parameters["b"] == pytest.approx(b_ml) + assert fit.parameters["inc"] == pytest.approx(88.8, abs=1e-6) + assert fit.errors["inc"] > 0 + + +def test_nested_fit_tracks_free_ars_with_internal_impact_parameter(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 101) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = elca.transit(time, prior) + + class DummySampler: + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + ml_values = prior.copy() + ml_values["ars"] = 12.3 + b_ml = float(elca.impact_parameter_from_inclination(ml_values, 88.8)) + sample_points = np.array( + [ + [0.100, 12.10, b_ml - 0.02, 0.0000], + [0.101, 12.20, b_ml - 0.01, 0.0002], + [0.099, 12.40, b_ml + 0.01, -0.0001], + [0.100, 12.50, b_ml + 0.02, 0.0001], + ] + ) + + monkeypatch.setattr(elca, "ReactiveNestedSampler", DummySampler) + monkeypatch.setattr( + elca, + "run_reactive_sampler", + lambda *args, **kwargs: { + "maximum_likelihood": {"point": np.array([0.100, 12.30, b_ml, 0.0])}, + "posterior": { + "stdev": np.array([0.005, 0.1, 0.02, 0.0005]), + "errlo": np.array([-0.005, -0.1, -0.02, -0.0005]), + "errup": np.array([0.005, 0.1, 0.02, 0.0005]), + }, + "weighted_samples": { + "points": sample_points, + "logl": np.array([-4.0, -3.0, -3.2, -3.8]), + }, + "samples": sample_points.copy(), + }, + ) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "ars": [11.5, 12.5], "inc": [87.0, 89.5], "tmid": [-0.005, 0.005]}, + mode="ns", + verbose=False, + ) + + assert fit.sampled_keys == ["rprs", "ars", "b", "tmid"] + assert fit.parameters["ars"] == pytest.approx(12.3, abs=1e-12) + assert fit.parameters["inc"] == pytest.approx(88.8, abs=1e-6) + assert fit.sample_bounds["b"] == pytest.approx([0.0, 1.12]) + + +def test_nested_fit_replaces_degenerate_ultranest_errors_from_loglike_neighborhood(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 101) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = elca.transit(time, prior) + + class DummySampler: + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + sample_points = np.array( + [ + [0.095, -0.0010], + [0.097, -0.0008], + [0.099, -0.0003], + [0.100, 0.0000], + [0.101, 0.0002], + [0.103, 0.0005], + [0.105, 0.0008], + [0.106, 0.0010], + [0.120, 0.0030], + [0.080, -0.0030], + ], + dtype=float, + ) + logl = np.array([-0.4, -0.3, -0.1, 0.0, -0.1, -0.2, -0.3, -0.4, -2.0, -3.0]) + + monkeypatch.setattr(elca, "ReactiveNestedSampler", DummySampler) + monkeypatch.setattr( + elca, + "run_reactive_sampler", + lambda *args, **kwargs: { + "maximum_likelihood": {"point": np.array([0.100, 0.0])}, + "posterior": { + "stdev": np.array([1e-15, 1e-15]), + "errlo": np.array([0.100, 0.0]), + "errup": np.array([0.100, 0.0]), + }, + "weighted_samples": { + "points": sample_points, + "logl": logl, + }, + "samples": np.repeat(np.array([[0.100, 0.0]]), 10, axis=0), + }, + ) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.0, 0.2], "tmid": [-0.01, 0.01]}, + mode="ns", + verbose=False, + ) + + assert fit.errors["rprs"] > 1e-3 + assert fit.errors["tmid"] > 1e-4 + assert set(fit.ultranest_error_fallbacks) == {"rprs", "tmid"} + assert fit.ultranest_error_fallbacks["rprs"]["sample_count"] == 8 + + +def test_nested_fit_replaces_prior_width_like_error_with_local_likelihood_width(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + fit.prior = make_prior() + fit.bounds = {"rprs": [0.0, 0.3]} + fit.mode = "ns" + fit.use_impactparameter_rather_than_inclination_to_fit = True + fit.fixed_parameter_errors = {} + + center = 0.152 + broad_points = np.linspace(0.0, 0.3, 40) + local_points = center + np.linspace(-0.006, 0.006, 17) + points = np.concatenate([broad_points, local_points])[:, None] + broad_logl = np.full(broad_points.shape, -100.0) + local_logl = -0.5 * ((local_points - center) / 0.0038) ** 2 + logl = np.concatenate([broad_logl, local_logl]) + fit.results = { + "maximum_likelihood": {"point": np.array([center])}, + "posterior": { + "stdev": np.array([0.082]), + "errlo": np.array([-0.082]), + "errup": np.array([0.082]), + }, + "weighted_samples": { + "points": points, + "logl": logl, + }, + "samples": points.copy(), + } + + fit._finalize_ultranest_fit_results( + ["rprs"], + ["rprs"], + lambda point: {"rprs": float(point[0])}, + ) + + fallback = fit.ultranest_error_fallbacks["rprs"] + assert fit.errors["rprs"] < 0.01 + assert fit.errors["rprs"] == pytest.approx(fallback["error"]) + assert fallback["reported_error"] == pytest.approx(0.082) + assert fallback["reason"] == "posterior_summary_inflated_relative_to_local_fit" + assert fallback["delta_chi2"] <= 1.0 + + +def test_nested_fit_duration_prior_penalizes_wrong_transit_length(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(0.20, 0.30, 51) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = np.ones_like(time) + data[0] += 1e-4 + + class DummySampler: + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + captured = {} + good_point = np.array([prior["rprs"], prior["ars"], prior["inc"], prior["tmid"]], dtype=float) + bad_point = np.array([prior["rprs"], 30.0, prior["inc"], prior["tmid"]], dtype=float) + + monkeypatch.setattr(elca, "ReactiveNestedSampler", DummySampler) + + def fake_run_reactive_sampler(sampler, *args, **kwargs): + loglike = sampler.args[1] + captured["good"] = float(loglike(good_point)) + captured["bad"] = float(loglike(bad_point)) + return { + "maximum_likelihood": {"point": good_point.copy()}, + "posterior": { + "stdev": np.array([0.001, 0.1, 0.05, 0.0001]), + "errlo": np.array([-0.001, -0.1, -0.05, -0.0001]), + "errup": np.array([0.001, 0.1, 0.05, 0.0001]), + }, + "weighted_samples": { + "points": np.vstack([good_point, bad_point]), + "logl": np.array([captured["good"], captured["bad"]]), + }, + "samples": np.vstack([good_point, bad_point]), + } + + monkeypatch.setattr(elca, "run_reactive_sampler", fake_run_reactive_sampler) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "ars": [10.0, 35.0], "inc": [88.5, 89.5], "tmid": [-0.005, 0.005]}, + mode="ns", + verbose=False, + use_impactparameter_rather_than_inclination_to_fit=False, + duration_prior={ + "applied": True, + "expected_duration": elca.transit_duration(prior), + "sigma_log_duration": 0.05, + }, + ) + + expected_penalty = -0.5 * ( + np.log(elca.transit_duration({"per": prior["per"], "rprs": prior["rprs"], "ars": 30.0, "inc": prior["inc"], "ecc": prior["ecc"], "omega": prior["omega"]}) / elca.transit_duration(prior)) + / 0.05 + ) ** 2 + + assert fit.parameters["ars"] == pytest.approx(prior["ars"], abs=1e-12) + assert captured["good"] > captured["bad"] + assert (captured["bad"] - captured["good"]) == pytest.approx(expected_penalty, rel=1e-6, abs=1e-6) + + +def test_nested_fit_duration_prior_returns_finite_floor_for_invalid_geometry(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + prior = make_prior() + time = np.linspace(-0.03, 0.03, 51) + airmass = np.zeros_like(time) + dataerr = np.full_like(time, 1e-3) + data = elca.transit(time, prior) + + class DummySampler: + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + captured = {} + good_point = np.array([prior["rprs"], prior["ars"], prior["inc"], prior["tmid"]], dtype=float) + invalid_point = np.array([prior["rprs"], 50.0, 80.0, prior["tmid"]], dtype=float) + + monkeypatch.setattr(elca, "ReactiveNestedSampler", DummySampler) + + def fake_run_reactive_sampler(sampler, *args, **kwargs): + loglike = sampler.args[1] + prior_transform = sampler.args[2] + captured["invalid"] = float(loglike(invalid_point)) + captured["vector"] = np.asarray(loglike(np.vstack([good_point, invalid_point])), dtype=float) + captured["transformed"] = prior_transform(np.full((2, 4), 0.5, dtype=float)) + return { + "maximum_likelihood": {"point": good_point.copy()}, + "posterior": { + "stdev": np.array([0.001, 0.1, 0.05, 0.0001]), + "errlo": np.array([-0.001, -0.1, -0.05, -0.0001]), + "errup": np.array([0.001, 0.1, 0.05, 0.0001]), + }, + "weighted_samples": { + "points": np.vstack([good_point, invalid_point]), + "logl": captured["vector"], + }, + "samples": np.vstack([good_point, invalid_point]), + } + + monkeypatch.setattr(elca, "run_reactive_sampler", fake_run_reactive_sampler) + + fit = elca.lc_fitter( + time, + data, + dataerr, + airmass, + prior.copy(), + {"rprs": [0.08, 0.12], "ars": [10.0, 50.0], "inc": [80.0, 89.5], "tmid": [-0.005, 0.005]}, + mode="ns", + verbose=False, + use_impactparameter_rather_than_inclination_to_fit=False, + duration_prior={ + "applied": True, + "expected_duration": elca.transit_duration(prior), + "sigma_log_duration": 0.05, + }, + ) + + assert fit.parameters["ars"] == pytest.approx(prior["ars"], abs=1e-12) + assert np.isfinite(captured["invalid"]) + assert captured["invalid"] == pytest.approx(elca.BAD_LOG_LIKELIHOOD) + assert captured["vector"].shape == (2,) + assert np.all(np.isfinite(captured["vector"])) + assert captured["vector"][1] == pytest.approx(elca.BAD_LOG_LIKELIHOOD) + assert np.asarray(captured["transformed"]).shape == (2, 4) + + +def test_rprs_posterior_recenter_diagnostics_detect_upper_bound_clipping(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.mode = "ns" + fit.use_impactparameter_rather_than_inclination_to_fit = True + fit.prior = make_prior() + fit.bounds = {"rprs": [0.0, 0.15], "tmid": [-0.005, 0.005]} + fit.sampled_keys = ["rprs", "tmid"] + fit.sample_bounds = {"rprs": [0.0, 0.15], "tmid": [-0.005, 0.005]} + + rprs_samples = np.concatenate([ + np.linspace(0.090, 0.120, 12), + np.linspace(0.128, 0.149, 28), + ]) + tmid_samples = np.linspace(-2e-4, 2e-4, rprs_samples.size) + points = np.column_stack([rprs_samples, tmid_samples]) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-6.0, -3.0, rprs_samples.size), + }, + "samples": points.copy(), + } + + diagnostics = fit.get_parameter_posterior_recenter_diagnostics("rprs") + + assert diagnostics["clipped"] is True + assert diagnostics["edge"] == "upper" + assert diagnostics["mode"] > 0.13 + assert diagnostics["std"] > 0 + assert diagnostics["upper_edge_peak_fraction"] >= 0.20 + assert diagnostics["bounds"][0] >= 0.0 + assert diagnostics["bounds"][1] > 0.15 + + +def test_ars_posterior_recenter_diagnostics_detect_lower_bound_clipping(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.mode = "ns" + fit.use_impactparameter_rather_than_inclination_to_fit = True + fit.prior = make_prior() + fit.bounds = {"ars": [10.0, 15.0], "tmid": [-0.005, 0.005]} + fit.sampled_keys = ["ars", "tmid"] + fit.sample_bounds = {"ars": [10.0, 15.0], "tmid": [-0.005, 0.005]} + + ars_samples = np.concatenate([ + np.linspace(10.001, 10.040, 30), + np.linspace(10.060, 10.800, 12), + ]) + tmid_samples = np.linspace(-2e-4, 2e-4, ars_samples.size) + points = np.column_stack([ars_samples, tmid_samples]) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-6.0, -3.0, ars_samples.size), + }, + "samples": points.copy(), + } + + diagnostics = fit.get_parameter_posterior_recenter_diagnostics("ars") + + assert diagnostics["clipped"] is True + assert diagnostics["edge"] == "lower" + assert diagnostics["mode"] < 10.5 + assert diagnostics["std"] > 0 + assert diagnostics["lower_edge_peak_fraction"] >= 0.20 + assert diagnostics["bounds"][0] < 10.0 + assert diagnostics["bounds"][0] >= 0.0 + + +def test_rprs_posterior_recenter_diagnostics_ignores_upper_edge_below_twenty_percent(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.mode = "ns" + fit.use_impactparameter_rather_than_inclination_to_fit = True + fit.prior = make_prior() + fit.bounds = {"rprs": [0.0, 0.15], "tmid": [-0.005, 0.005]} + fit.sampled_keys = ["rprs", "tmid"] + fit.sample_bounds = {"rprs": [0.0, 0.15], "tmid": [-0.005, 0.005]} + + rprs_samples = np.concatenate([ + np.linspace(0.106, 0.119, 40), + np.linspace(0.120, 0.134, 15), + np.linspace(0.145, 0.149, 5), + ]) + tmid_samples = np.linspace(-2e-4, 2e-4, rprs_samples.size) + points = np.column_stack([rprs_samples, tmid_samples]) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-6.0, -3.0, rprs_samples.size), + }, + "samples": points.copy(), + } + + diagnostics = fit.get_parameter_posterior_recenter_diagnostics("rprs") + + assert diagnostics["clipped"] is False + assert diagnostics["edge"] is None + assert diagnostics["upper_edge_peak_fraction"] < 0.20 + assert diagnostics["bounds"] == pytest.approx([0.0, 0.15]) + assert "not treated as truncated" in diagnostics["reason"] + + +def test_rprs_posterior_recenter_diagnostics_ignores_lower_edge_below_twenty_percent(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.mode = "ns" + fit.use_impactparameter_rather_than_inclination_to_fit = True + fit.prior = make_prior() + fit.bounds = {"rprs": [0.0, 0.15], "tmid": [-0.005, 0.005]} + fit.sampled_keys = ["rprs", "tmid"] + fit.sample_bounds = {"rprs": [0.0, 0.15], "tmid": [-0.005, 0.005]} + + rprs_samples = np.concatenate([ + np.linspace(0.001, 0.005, 5), + np.linspace(0.016, 0.029, 15), + np.linspace(0.031, 0.044, 40), + ]) + tmid_samples = np.linspace(-2e-4, 2e-4, rprs_samples.size) + points = np.column_stack([rprs_samples, tmid_samples]) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-6.0, -3.0, rprs_samples.size), + }, + "samples": points.copy(), + } + + diagnostics = fit.get_parameter_posterior_recenter_diagnostics("rprs") + + assert diagnostics["clipped"] is False + assert diagnostics["edge"] is None + assert diagnostics["lower_edge_peak_fraction"] < 0.20 + assert diagnostics["bounds"] == pytest.approx([0.0, 0.15]) + assert "not treated as truncated" in diagnostics["reason"] + + +def test_plot_triangle_uses_direct_fitted_impact_parameter_axis(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + captured = {} + + def fake_corner(*args, **kwargs): + captured["points"] = args[0] + captured["labels"] = kwargs["labels"] + captured["range"] = kwargs["range"] + captured["titles"] = kwargs["titles"] + captured["truths"] = kwargs["truths"] + captured["label_kwargs"] = kwargs["label_kwargs"] + return "figure" + + monkeypatch.setattr(elca, "corner", fake_corner) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.125], + "inc": [84.0, 90.0], + "a0": [0.95, 1.05], + } + fit.prior = make_prior() + fit.sampled_keys = ["rprs", "b", "a0"] + fit.sample_bounds = { + "rprs": [0.0, 0.125], + "b": [0.0, 1.25434156], + "a0": [0.95, 1.05], + } + fit.sample_parameters = {"rprs": 0.10, "b": 0.314, "a0": 0.94962} + fit.sample_errors = {"rprs": 0.01, "b": 0.05, "a0": 0.00394} + fit.parameters = {"rprs": 0.10, "inc": 88.5, "a0": 0.94962} + fit.errors = {"rprs": 0.01, "inc": 0.75, "a0": 0.00394} + + points = np.array( + [ + [0.099, 0.300, 0.9501], + [0.101, 0.330, 0.9502], + [0.102, 0.290, 0.9515], + [0.098, 0.360, 0.9520], + [0.100, 0.314, 0.9508], + ] + ) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.array([-5.0, -4.0, -4.5, -5.5, -4.2]), + }, + "samples": points.copy(), + } + + fig = fit.plot_triangle() + + assert fig == "figure" + assert captured["labels"][1] == r"Impact parameter $b$" + assert captured["range"][1] == pytest.approx([0.0, 1.25434156]) + assert captured["points"].shape == (5, 3) + np.testing.assert_allclose(captured["points"][:, 1], points[:, 1]) + assert captured["truths"][1] == pytest.approx(fit.sample_parameters["b"]) + assert captured["titles"][1].startswith("b=") + assert "\ni=" in captured["titles"][1] + assert captured["label_kwargs"]["labelpad"] == 10 + + +def test_triangle_contour_levels_drop_duplicate_chi2_percentiles(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + chi2 = np.full(12, 42.0) + mask = np.ones(chi2.size, dtype=bool) + + levels = fit._triangle_contour_levels(chi2, mask, mask, mask) + + assert levels == [pytest.approx(42.0)] + + +def test_triangle_plot_sigma_window_ranges_clip_to_solved_point_uncertainties(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + payload = { + "ranges": [[0.0, 1.0], [0.0, 1.2], [0.95, 1.05]], + "mask_centers": [0.20, 0.80, 1.0], + "mask_errors": [0.02, 0.05, 0.001], + "display_points": np.array( + [ + [0.18, 0.75, 0.999], + [0.20, 0.80, 1.000], + [0.22, 0.85, 1.001], + ] + ), + } + + ranges = fit._triangle_plot_sigma_window_ranges(payload, sigma=5.0) + + assert ranges[0] == pytest.approx([0.10, 0.30]) + assert ranges[1] == pytest.approx([0.55, 1.05]) + assert ranges[2] == pytest.approx([0.995, 1.005]) + + +def test_plot_triangle_accepts_zoom_sigma(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + captured = {} + + def fake_corner(*args, **kwargs): + captured["range"] = kwargs["range"] + return "figure" + + monkeypatch.setattr(elca, "corner", fake_corner) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 1.0], + "inc": [84.0, 90.0], + } + fit.sample_bounds = { + "rprs": [0.0, 1.0], + "b": [0.0, 1.2], + } + fit.sampled_keys = ["rprs", "b"] + fit.prior = make_prior() + fit.parameters = {"rprs": 0.20, "inc": 86.0} + fit.errors = {"rprs": 0.02, "inc": 0.5} + fit.sample_parameters = {"rprs": 0.20, "b": 0.80} + fit.sample_errors = {"rprs": 0.02, "b": 0.05} + points = np.column_stack([ + np.linspace(0.18, 0.22, 40), + np.linspace(0.75, 0.85, 40), + ]) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-4.0, -1.0, points.shape[0]), + }, + "samples": points.copy(), + } + + fig = fit.plot_triangle(zoom_sigma=5.0) + + assert fig == "figure" + assert captured["range"][0][0] > 0.0 + assert captured["range"][0][1] < 1.0 + assert captured["range"][1][0] > 0.0 + assert captured["range"][1][1] < 1.2 + + +def test_triangle_payload_recenter_uses_visible_zoom_peak(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + ars_values = np.concatenate([ + np.linspace(5.22, 5.30, 60), + np.linspace(6.45, 6.55, 20), + np.linspace(8.0, 9.0, 20), + ]) + payload = { + "sampled_keys": ["ars"], + "display_points": ars_values[:, None], + "display_weights": None, + "ranges": [[5.0, 7.0]], + "titles": ["6.0 +/- 1.0"], + "truths": [6.0], + "mask_centers": [6.0], + "mask_errors": [1.0], + "display_spec": None, + "geometry_summary": {}, + } + + updated = fit._recenter_triangle_plot_payload_for_visible_ranges(payload) + + assert updated["truths"][0] == pytest.approx(5.3) + assert updated["mask_centers"][0] == pytest.approx(5.3) + assert updated["titles"][0].startswith("5.3 +/-") + + +def test_triangle_payload_expands_degenerate_error_ranges_to_sample_cloud(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.2], + "tmid": [-0.01, 0.01], + } + fit.sample_bounds = dict(fit.bounds) + fit.sampled_keys = ["rprs", "tmid"] + fit.prior = make_prior() + fit.parameters = {"rprs": 0.100, "tmid": 0.0} + fit.errors = {"rprs": 1e-15, "tmid": 1e-15} + fit.sample_parameters = dict(fit.parameters) + fit.sample_errors = dict(fit.errors) + points = np.column_stack( + [ + np.linspace(0.050, 0.150, 50), + np.linspace(-0.004, 0.004, 50), + ] + ) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-4.0, -1.0, points.shape[0]), + }, + "samples": np.repeat(np.array([[0.100, 0.0]]), points.shape[0], axis=0), + } + + payload = fit._get_triangle_plot_payload() + + assert payload["ranges"][0][0] <= 0.052 + assert payload["ranges"][0][1] >= 0.148 + assert payload["ranges"][1][0] <= -0.0038 + assert payload["ranges"][1][1] >= 0.0038 + + +def test_triangle_payload_titles_follow_weighted_posterior_display_estimate(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.34], + "a0": [0.95, 1.05], + } + fit.sample_bounds = dict(fit.bounds) + fit.sampled_keys = ["rprs", "a0"] + fit.prior = make_prior() + fit.parameters = {"rprs": 0.33796, "a0": 1.0} + fit.errors = {"rprs": 0.09150, "a0": 0.001} + fit.sample_parameters = dict(fit.parameters) + fit.sample_errors = dict(fit.errors) + rprs_samples = np.concatenate([ + np.linspace(0.108, 0.122, 20), + np.linspace(0.318, 0.338, 80), + ]) + weights = np.concatenate([ + np.ones(20, dtype=float), + np.full(80, 0.01, dtype=float), + ]) + points = np.column_stack([rprs_samples, np.linspace(0.998, 1.002, rprs_samples.size)]) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-4.0, -1.0, points.shape[0]), + "weights": weights, + }, + "samples": points.copy(), + } + + payload = fit._get_triangle_plot_payload() + + assert payload["titles"][0] == "0.1190 +/- 0.0089" + assert payload["truths"][0] == pytest.approx(0.1186, abs=5e-4) + np.testing.assert_allclose(payload["display_weights"], weights) + + +def test_plot_triangle_passes_ultranest_weights_to_visible_histograms(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + captured = {} + + def fake_corner(*args, **kwargs): + captured["weights"] = kwargs["weights"] + captured["truths"] = kwargs["truths"] + captured["data_kwargs"] = kwargs["data_kwargs"] + return "figure" + + monkeypatch.setattr(elca, "corner", fake_corner) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.2], + "a0": [0.95, 1.05], + } + fit.sample_bounds = dict(fit.bounds) + fit.sampled_keys = ["rprs", "a0"] + fit.prior = make_prior() + fit.parameters = {"rprs": 0.1, "a0": 1.0} + fit.errors = {"rprs": 0.01, "a0": 0.001} + fit.sample_parameters = dict(fit.parameters) + fit.sample_errors = dict(fit.errors) + points = np.column_stack([ + np.linspace(0.09, 0.11, 12), + np.linspace(0.998, 1.002, 12), + ]) + weights = np.linspace(1.0, 2.0, points.shape[0]) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-4.0, -1.0, points.shape[0]), + "weights": weights, + }, + "samples": points.copy(), + } + + fig = fit.plot_triangle() + + assert fig == "figure" + np.testing.assert_allclose(captured["weights"], weights) + np.testing.assert_allclose(captured["truths"], [0.1018961, 1.00037922], rtol=1e-6) + assert captured["data_kwargs"]["s"] == pytest.approx(1.6) + assert captured["data_kwargs"]["alpha"] == pytest.approx(0.38) + + +def test_triangle_payload_expands_sparse_visible_ranges_to_sample_cloud(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.2], + "a0": [0.95, 1.05], + } + fit.sample_bounds = dict(fit.bounds) + fit.sampled_keys = ["rprs", "a0"] + fit.prior = make_prior() + fit.parameters = {"rprs": 0.100, "a0": 1.0} + fit.errors = {"rprs": 0.01, "a0": 1e-4} + fit.sample_parameters = dict(fit.parameters) + fit.sample_errors = dict(fit.errors) + points = np.column_stack( + [ + np.linspace(0.090, 0.110, 100), + np.linspace(0.980, 1.020, 100), + ] + ) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-4.0, -1.0, points.shape[0]), + }, + "samples": points.copy(), + } + + payload = fit._get_triangle_plot_payload() + + assert payload["ranges"][1][0] <= 0.981 + assert payload["ranges"][1][1] >= 1.019 + + +def test_triangle_payload_uses_tested_rprs_range_when_posterior_is_narrow(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.2], + "a0": [0.95, 1.05], + } + fit.sample_bounds = dict(fit.bounds) + fit.sampled_keys = ["rprs", "a0"] + fit.prior = make_prior() + fit.parameters = {"rprs": 0.100, "a0": 1.0} + fit.errors = {"rprs": 0.001, "a0": 0.001} + fit.sample_parameters = dict(fit.parameters) + fit.sample_errors = dict(fit.errors) + points = np.column_stack( + [ + np.linspace(0.090, 0.110, 120), + np.linspace(0.998, 1.002, 120), + ] + ) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-4.0, -1.0, points.shape[0]), + }, + "samples": points.copy(), + } + + payload = fit._get_triangle_plot_payload() + rprs_range = payload["ranges"][0] + plot_bins = int(max(1, np.sqrt(points.shape[0]))) + lower_fraction, upper_fraction, _ = fit._histogram_edge_peak_fractions( + points[:, 0], + rprs_range, + plot_bins, + ) + + assert rprs_range == pytest.approx([0.0, 0.2]) + assert lower_fraction < elca.TRIANGLE_PLOT_EDGE_PEAK_FRACTION_MAX + assert upper_fraction < elca.TRIANGLE_PLOT_EDGE_PEAK_FRACTION_MAX + + +def test_triangle_payload_expands_rprs_lower_edge_past_narrow_recorded_sample_bounds(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.36], + "a0": [0.95, 1.05], + } + fit.sample_bounds = { + "rprs": [0.104, 0.184], + "a0": [0.95, 1.05], + } + fit.sampled_keys = ["rprs", "a0"] + fit.prior = make_prior() + fit.parameters = {"rprs": 0.144, "a0": 1.0} + fit.errors = {"rprs": 0.008, "a0": 0.001} + fit.sample_parameters = dict(fit.parameters) + fit.sample_errors = dict(fit.errors) + rprs_samples = np.concatenate([ + np.linspace(0.104, 0.120, 80), + np.linspace(0.120, 0.180, 20), + ]) + points = np.column_stack([ + rprs_samples, + np.linspace(0.998, 1.002, rprs_samples.size), + ]) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-4.0, -1.0, points.shape[0]), + }, + "samples": points.copy(), + } + + payload = fit._get_triangle_plot_payload() + + assert payload["ranges"][0][0] < 0.08 + assert payload["truths"][0] < 0.13 + assert not payload["titles"][0].startswith("0.144") + + +def test_triangle_payload_keeps_direct_impact_parameter_full_sample_range(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.2], + "inc": [84.0, 90.0], + "a0": [0.95, 1.05], + } + fit.sampled_keys = ["rprs", "b", "a0"] + fit.sample_bounds = { + "rprs": [0.0, 0.2], + "b": [0.0, 1.2], + "a0": [0.95, 1.05], + } + fit.prior = make_prior() + fit.sample_parameters = {"rprs": 0.10, "b": 0.30, "a0": 1.0} + fit.sample_errors = {"rprs": 0.01, "b": 0.01, "a0": 0.001} + fit.parameters = {"rprs": 0.10, "inc": 88.6, "a0": 1.0} + fit.errors = {"rprs": 0.01, "inc": 0.75, "a0": 0.001} + points = np.column_stack( + [ + np.linspace(0.090, 0.110, 100), + np.linspace(0.10, 0.80, 100), + np.linspace(0.998, 1.002, 100), + ] + ) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-4.0, -1.0, points.shape[0]), + }, + "samples": points.copy(), + } + + payload = fit._get_triangle_plot_payload() + + assert payload["labels"][1] == r"Impact parameter $b$" + assert payload["ranges"][1] == pytest.approx([0.0, 1.2]) + assert payload["display_points"].shape == points.shape + np.testing.assert_allclose(payload["display_points"][:, 1], points[:, 1]) + assert payload["truths"][1] == pytest.approx(fit.sample_parameters["b"]) + assert payload["display_spec"]["mirror"] is False + + +def test_triangle_payload_uses_full_b_range_for_direct_impact_parameter(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.2], + "inc": [84.0, 90.0], + "a0": [0.95, 1.05], + } + fit.sampled_keys = ["rprs", "b", "a0"] + fit.sample_bounds = { + "rprs": [0.0, 0.2], + "b": [0.0, 1.2], + "a0": [0.95, 1.05], + } + fit.prior = make_prior() + fit.sample_parameters = {"rprs": 0.10, "b": 0.856, "a0": 1.0} + fit.sample_errors = {"rprs": 0.01, "b": 0.002, "a0": 0.001} + fit.parameters = {"rprs": 0.10, "inc": 85.0, "a0": 1.0} + fit.errors = {"rprs": 0.01, "inc": 2.7, "a0": 0.001} + b_samples = np.linspace(0.846, 0.866, 100) + points = np.column_stack([ + np.linspace(0.090, 0.110, b_samples.size), + b_samples, + np.linspace(0.998, 1.002, b_samples.size), + ]) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.linspace(-4.0, -1.0, points.shape[0]), + }, + "samples": points.copy(), + } + + payload = fit._get_triangle_plot_payload() + reference_values = [reference["value"] for reference in payload["display_spec"]["reference_lines"]] + + assert payload["labels"][1] == r"Impact parameter $b$" + assert payload["ranges"][1] == pytest.approx([0.0, 1.2]) + assert payload["truths"][1] == pytest.approx(0.856) + assert payload["display_spec"]["mirror"] is False + assert reference_values == pytest.approx([1.0, 1.10]) + + +def test_triangle_payload_tracks_left_and_right_geometry_branches_for_inclination(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.125], + "inc": [84.0, 90.0], + "a0": [0.95, 1.05], + } + fit.prior = make_prior() + fit.parameters = {"rprs": 0.10, "inc": 88.42, "a0": 0.94962} + fit.errors = {"rprs": 0.01, "inc": 0.75, "a0": 0.00394} + points = np.array( + [ + [0.099, 88.30, 0.9501], + [0.101, 88.55, 0.9502], + [0.102, 88.10, 0.9515], + [0.098, 88.70, 0.9520], + [0.100, 88.40, 0.9508], + ] + ) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.array([-5.0, -4.0, -4.5, -5.5, -4.2]), + }, + "samples": points.copy(), + } + + payload = fit._get_triangle_plot_payload() + + np.testing.assert_allclose( + payload["geometry_overlay"]["left_mirrored"], + np.array([-0.12, -0.32, -0.02, 0.12, 0.32, 0.02]), + atol=1e-12, + ) + np.testing.assert_allclose( + payload["geometry_overlay"]["right_mirrored"], + np.array([0.13, 0.28, -0.13, -0.28]), + atol=1e-12, + ) + + +def test_triangle_payload_skips_mirrored_overlay_for_direct_impact_parameter(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fit.ns_type = "ultranest" + fit.bounds = { + "rprs": [0.0, 0.125], + "inc": [84.0, 90.0], + "a0": [0.95, 1.05], + } + fit.prior = make_prior() + fit.sampled_keys = ["rprs", "b", "a0"] + fit.sample_bounds = { + "rprs": [0.0, 0.125], + "b": [0.0, 1.25434156], + "a0": [0.95, 1.05], + } + fit.sample_parameters = {"rprs": 0.10, "b": 0.314, "a0": 0.94962} + fit.sample_errors = {"rprs": 0.01, "b": 0.05, "a0": 0.00394} + fit.parameters = {"rprs": 0.10, "inc": 88.5, "a0": 0.94962} + fit.errors = {"rprs": 0.01, "inc": 0.75, "a0": 0.00394} + points = np.array( + [ + [0.099, 0.300, 0.9501], + [0.101, 0.330, 0.9502], + [0.102, 0.290, 0.9515], + [0.098, 0.360, 0.9520], + [0.100, 0.314, 0.9508], + ] + ) + fit.results = { + "weighted_samples": { + "points": points, + "logl": np.array([-5.0, -4.0, -4.5, -5.5, -4.2]), + }, + "samples": points.copy(), + } + + payload = fit._get_triangle_plot_payload() + + assert payload["geometry_overlay"] is None + assert payload["display_spec"]["mirror"] is False + assert payload["ranges"][1] == pytest.approx([0.0, 1.25434156]) + np.testing.assert_allclose(payload["display_points"][:, 1], points[:, 1]) + + +def test_triangle_geometry_curves_fall_back_to_surviving_branch(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + curves = fit._build_triangle_plot_geometry_curves( + { + "left_count": 0, + "right_count": 4, + "left_mirrored": np.array([], dtype=float), + "right_mirrored": np.array([0.05, 0.10, 0.15, -0.05, -0.10, -0.15], dtype=float), + }, + [-0.3, 0.3], + 31, + ) + + np.testing.assert_allclose(curves["main_curve"], fit._smooth_triangle_plot_counts(curves["right_curve"])) + assert np.allclose(curves["left_curve"], 0.0) + + +def test_triangle_geometry_overlay_reuses_shared_title_and_label_kwargs(monkeypatch, tmp_path): + elca = load_elca_with_stubs(monkeypatch, tmp_path) + fit = elca.lc_fitter.__new__(elca.lc_fitter) + + fig, axes = plt.subplots(2, 2) + payload = { + "sampled_keys": ["rprs", "b"], + "display_points": np.zeros((10, 2)), + "display_spec": { + "index": 1, + "center": 0.92, + "reference_lines": [ + {"value": 1.0, "linestyle": ":", "color": "#707070"}, + {"value": 1.10, "linestyle": "-.", "color": "#a35d00"}, + ], + }, + "geometry_overlay": { + "index": 1, + "left_count": 2, + "right_count": 2, + "left_mirrored": np.array([-0.1, 0.1]), + "right_mirrored": np.array([-0.2, 0.2]), + }, + "ranges": [[0.0, 0.1], [-0.3, 0.3]], + "titles": ["rprs", "b=0.32 +/- 0.18\ni=88.5 +/- 1.35 deg"], + "labels": ["rprs", r"$\Delta b$"], + } + + fit._overlay_triangle_plot_geometry_histograms( + fig, + payload, + title_kwargs={"loc": "left", "pad": 4, "fontsize": 12}, + label_kwargs={"labelpad": 10}, + ) + + ax = axes[1, 1] + assert ax.title.get_fontsize() == pytest.approx(12.0) + assert ax.xaxis.label.get_text() == r"$\Delta b$" + assert ax.xaxis.labelpad == pytest.approx(10.0) + reference_offsets = [] + for line in ax.lines: + xdata = np.asarray(line.get_xdata(), dtype=float) + if xdata.size == 2 and np.allclose(xdata, xdata[0]) and line.get_linestyle() in (":", "-."): + reference_offsets.append(float(xdata[0])) + assert sorted(reference_offsets) == pytest.approx([-0.18, -0.08, 0.08, 0.18]) + plt.close(fig) diff --git a/tests/test_ephemeris_validation.py b/tests/test_ephemeris_validation.py new file mode 100644 index 00000000..9154efe8 --- /dev/null +++ b/tests/test_ephemeris_validation.py @@ -0,0 +1,92 @@ +import numpy as np +import pytest + +from exotic.exotic import resolve_required_transit_ephemeris + + +def test_required_ephemeris_keeps_valid_initialization_values_without_archive_lookup(): + lookup_called = False + + def archive_lookup(): + nonlocal lookup_called + lookup_called = True + return {'pPer': 9.0, 'midT': 2469999.0} + + result = resolve_required_transit_ephemeris( + {'pName': 'Example b', 'pPer': 2.5, 'midT': 2460000.25}, + archive_lookup=archive_lookup, + ) + + assert result['pPer'] == 2.5 + assert result['midT'] == 2460000.25 + assert lookup_called is False + + +@pytest.mark.parametrize( + ('initial_values', 'expected_period', 'expected_tmid'), + [ + ({'pPer': None, 'midT': 2460000.25}, 2.5, 2460000.25), + ({'pPer': 0.0, 'midT': 2460000.25}, 2.5, 2460000.25), + ({'pPer': np.nan, 'midT': 2460000.25}, 2.5, 2460000.25), + ({'pPer': True, 'midT': 2460000.25}, 2.5, 2460000.25), + ({'pPer': 2.5, 'midT': None}, 2.5, 2460000.25), + ({'pPer': 2.5, 'midT': 0.0}, 2.5, 2460000.25), + ({'pPer': 2.5, 'midT': np.nan}, 2.5, 2460000.25), + ], +) +def test_required_ephemeris_fills_only_invalid_values_from_archive( + initial_values, expected_period, expected_tmid): + result = resolve_required_transit_ephemeris( + {'pName': 'Example b', **initial_values}, + archive_planet_dict={ + 'pPer': 2.5, + 'pPerUnc': 0.001, + 'midT': 2460000.25, + 'midTUnc': 0.002, + }, + ) + + assert result['pPer'] == expected_period + assert result['midT'] == expected_tmid + + +def test_required_ephemeris_copies_archive_uncertainty_with_fallback_value(): + result = resolve_required_transit_ephemeris( + { + 'pName': 'Example b', + 'pPer': None, + 'pPerUnc': None, + 'midT': 2460000.25, + 'midTUnc': 0.005, + }, + archive_planet_dict={ + 'pPer': 2.5, + 'pPerUnc': 0.001, + 'midT': 2461111.0, + 'midTUnc': 0.002, + }, + ) + + assert result['pPer'] == 2.5 + assert result['pPerUnc'] == 0.001 + assert result['midT'] == 2460000.25 + assert result['midTUnc'] == 0.005 + + +def test_required_ephemeris_fails_before_reduction_when_archive_values_are_unusable(): + with pytest.raises(ValueError, match=r"Cannot start EXOTIC reduction.*pPer.*midT"): + resolve_required_transit_ephemeris( + {'pName': 'Example b', 'pPer': None, 'midT': 0.0}, + archive_planet_dict={'pPer': np.nan, 'midT': None}, + ) + + +def test_required_ephemeris_reports_archive_lookup_failure(): + def archive_lookup(): + raise RuntimeError('archive unavailable') + + with pytest.raises(ValueError, match=r"NASA Exoplanet Archive fallback failed.*archive unavailable"): + resolve_required_transit_ephemeris( + {'pName': 'Example b', 'pPer': None, 'midT': 2460000.25}, + archive_lookup=archive_lookup, + ) diff --git a/tests/test_exotic_proper_motion.py b/tests/test_exotic_proper_motion.py new file mode 100644 index 00000000..05e43974 --- /dev/null +++ b/tests/test_exotic_proper_motion.py @@ -0,0 +1,8667 @@ +import importlib +import importlib.util +import json +import sys +import types +from pathlib import Path +import numpy as np +import pytest +from astropy.wcs import WCS + + +def _module_available(name: str) -> bool: + try: + return importlib.util.find_spec(name) is not None + except Exception: + return False + + +def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: + if not _module_available(name): + sys.modules.setdefault(name, module) + + +fake_barycorrpy = types.ModuleType("barycorrpy") +fake_utc_tdb = types.ModuleType("barycorrpy.utc_tdb") +fake_utc_tdb.JDUTC_to_BJDTDB = lambda *args, **kwargs: None +fake_barycorrpy.utc_tdb = fake_utc_tdb +fake_astroalign = types.ModuleType("astroalign") +fake_astroalign.PIXEL_TOL = 1 +fake_astroquery = types.ModuleType("astroquery") +fake_astroquery_simbad = types.ModuleType("astroquery.simbad") +fake_astroquery_simbad.Simbad = type("Simbad", (), {}) +fake_astroquery_gaia = types.ModuleType("astroquery.gaia") +fake_astroquery_gaia.Gaia = type("Gaia", (), {}) +fake_imreg_dft = types.ModuleType("imreg_dft") +fake_colour_demosaicing = types.ModuleType("colour_demosaicing") +fake_colour_demosaicing.demosaicing_CFA_Bayer_bilinear = lambda *args, **kwargs: None +fake_photutils = types.ModuleType("photutils") +fake_photutils_aperture = types.ModuleType("photutils.aperture") +fake_photutils_aperture.CircularAperture = type("CircularAperture", (), {}) +fake_photutils_aperture.CircularAnnulus = type("CircularAnnulus", (), {}) +fake_photutils_detection = types.ModuleType("photutils.detection") +fake_photutils_detection.DAOStarFinder = type("DAOStarFinder", (), {}) +fake_ldtk = types.ModuleType("ldtk") +fake_ldtk.LDPSet = type("LDPSet", (), {}) +fake_ldtk.ldtk = types.SimpleNamespace(LDPSet=fake_ldtk.LDPSet) +fake_ldtk_ldmodel = types.ModuleType("ldtk.ldmodel") +fake_ldtk_ldmodel.LinearModel = type("LinearModel", (), {}) +fake_ldtk_ldmodel.QuadraticModel = type("QuadraticModel", (), {}) +fake_ldtk_ldmodel.NonlinearModel = type("NonlinearModel", (), {}) +fake_lmfit = types.ModuleType("lmfit") +fake_pylightcurve = types.ModuleType("pylightcurve") +fake_pylightcurve_models = types.ModuleType("pylightcurve.models") +fake_pylightcurve_exoplanet = types.ModuleType("pylightcurve.models.exoplanet_lc") +fake_pylightcurve_exoplanet.transit = lambda *args, **kwargs: None +fake_pyvo = types.ModuleType("pyvo") +fake_ultranest = types.ModuleType("ultranest") +fake_ultranest.ReactiveNestedSampler = type("ReactiveNestedSampler", (), {}) +fake_elca = types.ModuleType("exotic.api.elca") +fake_elca.lc_fitter = lambda *args, **kwargs: None +fake_elca.binner = lambda *args, **kwargs: None +fake_elca.transit = lambda *args, **kwargs: None +fake_elca.get_phase = lambda *args, **kwargs: None +fake_ld = types.ModuleType("exotic.api.ld") +fake_ld.LimbDarkening = type("LimbDarkening", (), {}) +fake_ld.ld_re_punct_p = lambda *args, **kwargs: None + +_set_stub_if_missing("astroalign", fake_astroalign) +_set_stub_if_missing("astroquery", fake_astroquery) +_set_stub_if_missing("astroquery.simbad", fake_astroquery_simbad) +_set_stub_if_missing("astroquery.gaia", fake_astroquery_gaia) +_set_stub_if_missing("imreg_dft", fake_imreg_dft) +_set_stub_if_missing("colour_demosaicing", fake_colour_demosaicing) +_set_stub_if_missing("photutils", fake_photutils) +_set_stub_if_missing("photutils.aperture", fake_photutils_aperture) +_set_stub_if_missing("photutils.detection", fake_photutils_detection) +_set_stub_if_missing("ldtk", fake_ldtk) +_set_stub_if_missing("ldtk.ldmodel", fake_ldtk_ldmodel) +_set_stub_if_missing("lmfit", fake_lmfit) +_set_stub_if_missing("pylightcurve", fake_pylightcurve) +_set_stub_if_missing("pylightcurve.models", fake_pylightcurve_models) +_set_stub_if_missing("pylightcurve.models.exoplanet_lc", fake_pylightcurve_exoplanet) +_set_stub_if_missing("pyvo", fake_pyvo) +_set_stub_if_missing("ultranest", fake_ultranest) +_set_stub_if_missing("barycorrpy", fake_barycorrpy) +_set_stub_if_missing("barycorrpy.utc_tdb", fake_utc_tdb) +try: + importlib.import_module("barycorrpy.utc_tdb") +except Exception: + sys.modules["barycorrpy"] = fake_barycorrpy + sys.modules["barycorrpy.utc_tdb"] = fake_utc_tdb +sys.modules.setdefault("exotic.api.elca", fake_elca) +sys.modules.setdefault("exotic.api.ld", fake_ld) + +from exotic.exotic import ( + APERTURE_MAX_FWHM_MULTIPLIER, + APERTURE_MIN_FWHM_MULTIPLIER, + APERTURE_SIGMA_MAX, + APERTURE_SIGMA_MIN, + GAUSSIAN_SIGMA_TO_FWHM, + adaptive_aperture_outlier_mask, + annotate_transit_qc_expected_values, + aperture_contains_overexposed_pixel, + auto_tune_aperture_sigma_grid, + build_aperture_correction_profile, + build_initial_ars_bounds, + build_single_transit_duration_prior, + build_target_fit_candidate_jobs, + build_time_rejection_diagnostic, + check_coordinates, + cheap_lightcurve_prescore, + centroid_offset_matches_reference, + choose_centroid_seed_position, + compute_star_aperture_grid, + compute_transit_qc_ktmf, + detect_aperture_correction_star_candidates, + transit_qc_residual_scatter_score, + transit_qc_residual_flatness_summary, + transit_qc_sampling_summary, + transit_qc_tmid_gaussianity_summary, + apply_comparison_star_suitability_outlier_rejection, + comparison_calibration_selection_reason, + comparison_candidate_triangle_plot_output_path, + comparison_candidate_fit_selection_reason, + comparison_star_coverage_summary, + comparison_star_stability_summary, + compute_photometry_noise_budget, + configure_windows_multiprocessing_main_spec, + deduplicate_comparison_star_coords, + diagnose_lightcurve_fit_inputs, + detrend_flux_on_out_of_transit_baseline, + alignment_candidate_quality_score, + aperture_estimation_comparison_stars, + aperture_frame_sigma_from_psf_data, + apply_raw_target_photometry_selection, + build_tracked_comparison_pool, + collapse_aperture_data_to_selected_grid_cell, + ensure_lightcurve_fit_failure_reason, + evaluate_lightcurve_candidate, + evaluate_transit_detection_qc, + finalize_comparison_candidate_full_reduction, + fit_lightcurve, + fit_final_lightcurve_with_oot_baseline_detrending, + fit_lightcurve_to_every_comparison_candidate, + fit_ranked_comparison_calibration_candidates, + get_final_fit_baseline_duration_multiplier, + get_multiprocess_bad_pixel_precheck_processes, + estimate_ephemeris_tmid_and_bounds, + estimate_tmid_and_bounds_with_eebls, + initialize_aperture_data_store, + is_adaptive_aperture_mode_enabled, + is_comp_star_required, + is_out_of_transit_baseline_detrending_enabled, + is_target_driven_comp_selection_enabled, + limited_ensemble_comparison_keys, + log_comparison_calibration_fit_attempt_summaries, + log_comparison_candidate_fit_summaries, + log_target_fit_candidate_summaries, + noise_budget_config_from_info, + normalize_flux_series_to_approximate_unity, + parse_overexposure_threshold_fraction, + parse_saturation_value, + saturation_value_from_header, + phase_bin_sigma_clip, + parse_deviation_from_expected_transit_in_qc_sigma, + parse_maximum_number_of_ensemble_comparisons_for_stellar_variability, + parse_maximum_number_of_ensemble_comparisons_for_transit, + prepare_final_fit_lightcurve_series, + prepare_lightcurve_fit_input_series, + project_comparison_radec_to_pixels, + psf_frame_quality_components, + psf_frame_quality_mask, + psf_solution_quality_score, + target_psf_shape_quality_components, + target_psf_shape_quality_mask, + target_comp_flux_scatter, + fitted_lightcurve_scatter_on_dataset, + populate_aperture_data_for_frame, + rank_comparison_candidate_preflight_plans, + refit_selected_fast_comparison_on_full_lightcurve, + representative_psf_sigma, + ranked_comparison_calibration_summaries, + resolve_sky_annulus_geometry, + run_target_driven_photometry_search, + resolve_frame_aperture_radii, + robust_flux_floor_mask, + robust_target_reference_flux_mask, + save_final_triangle_plot, + save_selected_photometry_debug_series, + select_comparison_calibrated_photometry, + select_alignment_candidate, + select_preferred_comparison_attempt, + should_keep_header_wcs_alignment, + should_prefer_pixel_values_over_wcs_for_target, + sigma_clip, + summarize_adaptive_aperture_usage, + summarize_prior_transit_coverage, + should_skip_airmass_fit, + should_require_apparent_magnitudes, + should_use_exactly_the_comps_provided, + should_use_eebls_to_initialize_tmid_and_bounds, + should_use_ensemble_photometry_for_stellar_variability, + should_photometer_fortuitous_variables, + should_reject_overexposed_stars, + should_fit_lightcurve_to_every_comparison_candidate, + should_detect_bad_pixels_before_photometry, + should_use_aperture_photometry, + should_use_aperture_corrections_and_full_image_fwhm, + should_exit_at_first_qc_pass_solution, + should_pick_comparison_by_eebls_snr, + should_stop_after_promising_partial_comparison_attempt, + should_use_psf_photometry, + should_skip_low_comparison_coverage_rejection, + should_use_fast_target_centroid, + should_use_deviation_from_expected_transit_in_qc, + update_coordinates_with_proper_motion, + zoomed_final_triangle_plot_output_path, +) + + +def test_save_final_triangle_plot_regenerates_even_when_selected_candidate_artifact_exists(tmp_path): + class DummyFigure: + def savefig(self, path): + Path(path).write_bytes(b"regenerated-final") + + class DummyFit: + def __init__(self): + self.called = False + + def plot_triangle(self): + self.called = True + return DummyFigure() + + planet_name = "TOI-1728 b" + observation_date = "2024-12-14" + source_dir = tmp_path / "comp6" + final_dir = tmp_path / "final" + source_temp = source_dir / "working_artifacts" + source_temp.mkdir(parents=True) + source_plot = source_temp / "Triangle_TOI-1728b_2024-12-14.png" + source_plot.write_bytes(b"stale-selected-comp-6") + + fit = DummyFit() + output_path = save_final_triangle_plot( + fit, + final_dir, + planet_name, + observation_date, + source_dir=source_dir, + ) + + assert output_path == final_dir / "Diagnostics" / "FinalTriangle_TOI-1728b_2024-12-14.png" + assert output_path.read_bytes() == b"regenerated-final" + assert (final_dir / "Diagnostics" / "Triangle_TOI-1728b_2024-12-14.png").read_bytes() == b"regenerated-final" + assert fit.called is True + + +def test_comparison_candidate_triangle_plot_uses_candidate_specific_name(tmp_path): + output_path = comparison_candidate_triangle_plot_output_path( + tmp_path / "comp7", + "WASP-80 b", + "2025-06-22", + 6, + ) + + assert output_path.name == "Comp7_Triangle_WASP-80b_2025-06-22.png" + assert output_path.parent == tmp_path / "comp7" / "working_artifacts" + + +def test_comparison_candidate_triangle_plot_uses_date_only_from_timestamp(tmp_path): + output_path = comparison_candidate_triangle_plot_output_path( + tmp_path / "comp7", + "XO-1/b", + "2026-05-06T19:51:13.964-0700", + 6, + ) + + assert output_path.name == "Comp7_Triangle_XO-1-b_2026-05-06.png" + + +def test_zoomed_final_triangle_plot_uses_named_artifact(tmp_path): + output_path = zoomed_final_triangle_plot_output_path( + tmp_path / "final", + "XO-1/b", + "2026-05-06T19:51:13.964-0700", + ) + + assert output_path == tmp_path / "final" / "Diagnostics" / "ZoomedTrianglePlot_XO-1-b_2026-05-06.png" + + +def test_save_final_triangle_plot_creates_zoomed_companion_when_supported(tmp_path): + class DummyFigure: + def __init__(self, content): + self.content = content + + def savefig(self, path): + Path(path).write_bytes(self.content) + + class DummyFit: + def __init__(self): + self.calls = [] + + def plot_triangle(self, plot_title=None, zoom_sigma=None): + self.calls.append({"plot_title": plot_title, "zoom_sigma": zoom_sigma}) + content = b"zoomed" if zoom_sigma == 5.0 else b"full" + return DummyFigure(content) + + fit = DummyFit() + output_path = save_final_triangle_plot( + fit, + tmp_path / "final", + "TOI-1728 b", + "2024-12-14", + source_dir=tmp_path / "comp4", + ) + + zoomed_output_path = zoomed_final_triangle_plot_output_path( + tmp_path / "final", + "TOI-1728 b", + "2024-12-14", + ) + assert output_path.read_bytes() == b"full" + assert zoomed_output_path.read_bytes() == b"zoomed" + assert fit.calls == [ + {"plot_title": "Final selected fit (comparison candidate #4)", "zoom_sigma": None}, + {"plot_title": "Final selected fit (comparison candidate #4) (5-sigma zoom)", "zoom_sigma": 5.0}, + ] + + +def test_save_final_triangle_plot_regenerates_when_selected_artifact_missing(tmp_path): + class DummyFigure: + def savefig(self, path): + Path(path).write_bytes(b"regenerated") + + class DummyFit: + def __init__(self): + self.called = False + + def plot_triangle(self): + self.called = True + return DummyFigure() + + fit = DummyFit() + output_path = save_final_triangle_plot( + fit, + tmp_path / "final", + "TOI-1728 b", + "2024-12-14", + source_dir=tmp_path / "missing-comp", + ) + + assert fit.called is True + assert output_path.read_bytes() == b"regenerated" + assert output_path.parent == tmp_path / "final" / "Diagnostics" + assert output_path.name == "FinalTriangle_TOI-1728b_2024-12-14.png" + assert ( + tmp_path + / "final" + / "Diagnostics" + / "Triangle_TOI-1728b_2024-12-14.png" + ).read_bytes() == b"regenerated" + + +def test_save_final_triangle_plot_labels_selected_candidate_when_supported(tmp_path): + class DummyFigure: + def savefig(self, path): + Path(path).write_bytes(b"regenerated") + + class DummyFit: + def __init__(self): + self.plot_title = None + + def plot_triangle(self, plot_title=None): + self.plot_title = plot_title + return DummyFigure() + + fit = DummyFit() + output_path = save_final_triangle_plot( + fit, + tmp_path / "final", + "TOI-1728 b", + "2024-12-14", + source_dir=tmp_path / "comp4", + ) + + assert output_path.read_bytes() == b"regenerated" + assert fit.plot_title == "Final selected fit (comparison candidate #4)" + + +def test_update_coordinates_handles_non_numeric_proper_motion_values(): + info = { + "ra": 10.0, + "dec": 20.0, + "dist": "", + "pm_ra": "nan-value", + "pm_dec": None, + } + + updated_ra, updated_dec = update_coordinates_with_proper_motion(info, 2459945.5) + + assert updated_ra == info["ra"] + assert updated_dec == info["dec"] + + +def test_update_coordinates_accepts_numeric_strings(): + info = { + "ra": 10.0, + "dec": 20.0, + "dist": "100", + "pm_ra": "10.5", + "pm_dec": "-5.25", + } + + updated_ra, updated_dec = update_coordinates_with_proper_motion(info, 2459945.5) + + assert isinstance(updated_ra, float) + assert isinstance(updated_dec, float) + + +def test_prepare_final_fit_lightcurve_series_uses_two_sided_modeled_oot(): + times = np.linspace(-0.04, 0.04, 9) + detrended = np.array([1.0, 1.0, 1.0, 0.99, 0.98, 0.99, 1.0, 1.0, 1.0], dtype=float) + fit = types.SimpleNamespace( + time=times, + data=detrended.copy(), + dataerr=np.full(times.shape, 0.01, dtype=float), + detrended=detrended.copy(), + detrendederr=np.full(times.shape, 0.01, dtype=float), + airmass_model=np.ones(times.shape, dtype=float), + transit=np.array([1.0, 1.0, 1.0, 0.99, 0.98, 0.99, 1.0, 1.0, 1.0], dtype=float), + parameters={"tmid": 0.0}, + ) + + prepared = prepare_final_fit_lightcurve_series(fit) + + assert prepared["applied"] is True + assert prepared["used_two_sided_oot"] is True + assert "modeled out-of-transit" in prepared["note"] + assert prepared["flux"] == pytest.approx(detrended) + assert np.all(prepared["unc"] > 0) + + +def test_prepare_final_fit_lightcurve_series_avoids_one_sided_oot_raw_flux_bias(): + times = np.linspace(-0.05, 0.05, 11) + detrended = 1.0 - 0.02 * np.exp(-0.5 * (times / 0.012) ** 2) + airmass_model = 1.0 + 2.0 * times + raw_flux = detrended * airmass_model + transit_model = np.where(times < 0.0, 1.0, 0.985) + fit = types.SimpleNamespace( + time=times, + data=raw_flux, + dataerr=np.full(times.shape, 0.01, dtype=float), + detrended=detrended.copy(), + detrendederr=np.full(times.shape, 0.01, dtype=float), + airmass_model=airmass_model, + transit=transit_model, + parameters={"tmid": 0.0}, + ) + + prepared = prepare_final_fit_lightcurve_series(fit) + old_oot_mask = transit_model == 1.0 + legacy_flux = raw_flux / np.nanmedian(raw_flux[old_oot_mask]) + expected_flux = detrended / np.nanmedian(detrended) + + assert prepared["applied"] is True + assert prepared["used_two_sided_oot"] is False + assert "only bracketed one side of transit" in prepared["note"] + assert prepared["flux"] == pytest.approx(expected_flux) + assert prepared["flux"][-1] != pytest.approx(legacy_flux[-1], abs=1e-3) + assert np.all(prepared["unc"] > 0) + + +def test_save_selected_photometry_debug_series_writes_stage_masks(tmp_path): + fit = types.SimpleNamespace( + selected_photometry_debug={ + "times": np.array([1.0, 2.0, 3.0], dtype=float), + "target_flux": np.array([10.0, 11.0, 12.0], dtype=float), + "comp_flux": np.array([5.0, 5.0, 6.0], dtype=float), + "raw_ratio": np.array([2.0, 2.2, 2.0], dtype=float), + "initial_sigma_keep_mask": np.array([True, False, True], dtype=bool), + "prefit_raw_ratio_keep_mask": np.array([True, True, True], dtype=bool), + "phase_clip_keep_mask_on_sigma_filtered": np.array([True, False], dtype=bool), + } + ) + + output_path = save_selected_photometry_debug_series(tmp_path, "Qatar-10 b", "20260420", fit) + + assert output_path is not None + assert output_path.exists() + + rows = np.loadtxt(output_path, delimiter=",", skiprows=1) + assert rows.shape == (3, 10) + assert np.isnan(rows[:, 4:7]).all() + assert rows[:, 7].astype(int).tolist() == [1, 0, 1] + assert rows[:, 8].astype(int).tolist() == [1, 1, 1] + assert rows[:, 9].astype(int).tolist() == [1, 0, 0] + + +def test_finalize_comparison_candidate_phase_clips_before_nested_fit(monkeypatch): + captured = {} + + def fake_lc_fitter(times, flux, unc, airmass, prior, bounds, jd_times=None, mode=None, **kwargs): + assert mode == "lm" + return types.SimpleNamespace( + residuals=np.linspace(-0.01, 0.01, len(times)), + phase=np.linspace(-0.5, 0.5, len(times)), + ) + + def fake_phase_clip(residuals, phase, sigma=3, bins=10): + mask = np.zeros(len(residuals), dtype=bool) + mask[3] = True + return mask + + def fake_final_fit( + times, + flux, + unc, + airmass, + prior, + bounds, + jd_times=None, + **kwargs, + ): + captured["times"] = np.asarray(times, dtype=float).copy() + captured["jd_times"] = np.asarray(jd_times, dtype=float).copy() + fit = types.SimpleNamespace( + time=np.asarray(times, dtype=float), + airmass=np.asarray(airmass, dtype=float), + data=np.asarray(flux, dtype=float), + dataerr=np.asarray(unc, dtype=float), + detrended=np.asarray(flux, dtype=float), + detrendederr=np.asarray(unc, dtype=float), + airmass_model=np.ones(len(times), dtype=float), + transit=np.ones(len(times), dtype=float), + phase=np.linspace(-0.5, 0.5, len(times)), + residuals=np.zeros(len(times), dtype=float), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a1": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a1": 0.01, "a2": 0.01}, + ) + return fit, np.asarray(flux, dtype=float), np.asarray(unc, dtype=float) + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr("exotic.exotic.phase_bin_sigma_clip", fake_phase_clip) + monkeypatch.setattr("exotic.exotic.fit_final_lightcurve_with_oot_baseline_detrending", fake_final_fit) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 0.09, 10) + result = finalize_comparison_candidate_full_reduction( + times, + np.full(10, 100.0, dtype=float), + np.full(10, 100.0, dtype=float), + np.linspace(1.0, 1.2, 10), + [0.1, 0.1, 0.1, 0.1], + { + "midT": 0.045, + "midTUnc": 0.001, + "pPer": 1.0, + "pPerUnc": 0.001, + "rprs": 0.1, + "aRs": 10.0, + "aRsUnc": 0.1, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + }, + jd_times=2460000.0 + times, + ) + + assert result["applied"] is True + assert captured["times"].tolist() == pytest.approx(np.delete(times, 3).tolist()) + assert result["source_indices"].tolist() == [0, 1, 2, 4, 5, 6, 7, 8, 9] + assert any( + diagnostic["stage"] == "Final-fit phase residual clip" + and diagnostic["dropped_point_count"] == 1 + for diagnostic in result["fit"].frame_filter_diagnostics + ) + assert result["fit"].selected_photometry_debug[ + "phase_clip_keep_mask_on_sigma_filtered" + ].tolist() == [True, True, True, False, True, True, True, True, True, True] + + +def test_finalize_comparison_candidate_can_disable_phase_residual_clip(monkeypatch): + captured = {"phase_clip_called": False} + + def fake_lc_fitter(times, flux, unc, airmass, prior, bounds, jd_times=None, mode=None, **kwargs): + return types.SimpleNamespace( + residuals=np.linspace(-0.01, 0.01, len(times)), + phase=np.linspace(-0.5, 0.5, len(times)), + ) + + def fake_phase_clip(residuals, phase, sigma=3, bins=10): + captured["phase_clip_called"] = True + mask = np.zeros(len(residuals), dtype=bool) + mask[3] = True + return mask + + def fake_final_fit(times, flux, unc, airmass, prior, bounds, jd_times=None, **kwargs): + captured["times"] = np.asarray(times, dtype=float).copy() + fit = types.SimpleNamespace( + time=np.asarray(times, dtype=float), + airmass=np.asarray(airmass, dtype=float), + data=np.asarray(flux, dtype=float), + dataerr=np.asarray(unc, dtype=float), + detrended=np.asarray(flux, dtype=float), + detrendederr=np.asarray(unc, dtype=float), + airmass_model=np.ones(len(times), dtype=float), + transit=np.ones(len(times), dtype=float), + phase=np.linspace(-0.5, 0.5, len(times)), + residuals=np.zeros(len(times), dtype=float), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a1": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a1": 0.01, "a2": 0.01}, + ) + return fit, np.asarray(flux, dtype=float), np.asarray(unc, dtype=float) + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr("exotic.exotic.phase_bin_sigma_clip", fake_phase_clip) + monkeypatch.setattr("exotic.exotic.fit_final_lightcurve_with_oot_baseline_detrending", fake_final_fit) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 0.09, 10) + result = finalize_comparison_candidate_full_reduction( + times, + np.full(10, 100.0, dtype=float), + np.full(10, 100.0, dtype=float), + np.linspace(1.0, 1.2, 10), + [0.1, 0.1, 0.1, 0.1], + { + "midT": 0.045, + "midTUnc": 0.001, + "pPer": 1.0, + "pPerUnc": 0.001, + "rprs": 0.1, + "aRs": 10.0, + "aRsUnc": 0.1, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + }, + jd_times=2460000.0 + times, + run_final_fit_phase_residual_clip=False, + ) + + assert result["applied"] is True + assert captured["phase_clip_called"] is False + assert captured["times"].tolist() == pytest.approx(times.tolist()) + assert result["source_indices"].tolist() == list(range(10)) + assert not any( + diagnostic["stage"] == "Final-fit phase residual clip" + and diagnostic["dropped_point_count"] > 0 + for diagnostic in result["fit"].frame_filter_diagnostics + ) + assert result["fit"].selected_photometry_debug[ + "phase_clip_keep_mask_on_sigma_filtered" + ].tolist() == [True] * 10 + + +def test_finalize_comparison_candidate_keeps_flux_aligned_after_final_fit_subsets_times(monkeypatch): + fit_calls = [] + initial_subset_mask = np.ones(38, dtype=bool) + initial_subset_mask[[1, 3, 5, 7, 9, 11]] = False + + def fake_lc_fitter(times, flux, unc, airmass, prior, bounds, jd_times=None, mode=None, **kwargs): + return types.SimpleNamespace( + residuals=np.zeros(len(times), dtype=float), + phase=np.linspace(-0.5, 0.5, len(times)), + ) + + def fake_final_fit(times, flux, unc, airmass, prior, bounds, jd_times=None, **kwargs): + times = np.asarray(times, dtype=float) + flux = np.asarray(flux, dtype=float) + unc = np.asarray(unc, dtype=float) + airmass = np.asarray(airmass, dtype=float) + fit_calls.append({ + "times": times.copy(), + "flux": flux.copy(), + "unc": unc.copy(), + "airmass": airmass.copy(), + }) + + if len(fit_calls) == 1: + keep_mask = initial_subset_mask + residuals = np.zeros(np.count_nonzero(keep_mask), dtype=float) + residuals[10] = 1.0 + else: + keep_mask = np.ones(len(times), dtype=bool) + residuals = np.zeros(len(times), dtype=float) + + retained_times = times[keep_mask] + retained_flux = flux[keep_mask] + retained_unc = unc[keep_mask] + retained_airmass = airmass[keep_mask] + fit = types.SimpleNamespace( + time=retained_times, + airmass=retained_airmass, + data=retained_flux, + dataerr=retained_unc, + detrended=retained_flux, + detrendederr=retained_unc, + airmass_model=np.ones(len(retained_times), dtype=float), + transit=np.ones(len(retained_times), dtype=float), + phase=np.linspace(-0.5, 0.5, len(retained_times)), + residuals=residuals, + parameters={ + "tmid": 0.5, + "rprs": 0.1, + "ars": 10.0, + "inc": 89.0, + "a0": 1.0, + "a1": 1.0, + "a2": 0.0, + }, + errors={ + "tmid": 0.001, + "rprs": 0.001, + "ars": 0.1, + "inc": 0.1, + "a0": 0.01, + "a1": 0.01, + "a2": 0.01, + }, + ) + return fit, retained_flux, retained_unc + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr("exotic.exotic.fit_final_lightcurve_with_oot_baseline_detrending", fake_final_fit) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 1.0, 38) + result = finalize_comparison_candidate_full_reduction( + times, + np.full(38, 100.0, dtype=float), + np.full(38, 100.0, dtype=float), + np.linspace(1.0, 1.2, 38), + [0.1, 0.1, 0.1, 0.1], + { + "midT": 0.5, + "midTUnc": 0.001, + "pPer": 1.0, + "pPerUnc": 0.001, + "rprs": 0.1, + "aRs": 10.0, + "aRsUnc": 0.1, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + }, + jd_times=2460000.0 + times, + run_fast_ultranest_before_final_run=False, + run_final_fit_phase_residual_clip=False, + run_final_residual_rejection=True, + use_eebls_to_initialize_tmid_and_bounds=False, + ) + + initially_retained_indices = np.flatnonzero(initial_subset_mask) + expected_source_indices = np.delete(initially_retained_indices, 10) + assert result["applied"] is True + assert [len(call["times"]) for call in fit_calls] == [38, 31] + assert all(call["times"].shape == call["flux"].shape == call["unc"].shape for call in fit_calls) + assert result["source_indices"].tolist() == expected_source_indices.tolist() + assert result["good_times"].tolist() == pytest.approx(times[expected_source_indices].tolist()) + assert all( + len(result[key]) == 31 + for key in ( + "good_times", + "good_flux", + "good_unc", + "good_airmass", + "good_jd_times", + "good_target_flux", + "good_comp_flux", + "good_target_flux_error", + "good_comp_flux_error", + "source_indices", + ) + ) + + +def test_detrend_flux_on_out_of_transit_baseline_falls_back_to_prior_ephemeris(): + times = np.linspace(-0.08, 0.08, 17) + baseline = 1.0 + 0.25 * times + transit = np.ones_like(times) + transit[(times >= 0.03) & (times <= 0.05)] = 0.99 + flux = baseline.copy() + unc = np.full_like(times, 0.01) + fit = types.SimpleNamespace( + transit=transit, + parameters={"tmid": 0.10}, + ) + prior = { + "tmid": 0.0, + "per": 1.0, + "rprs": 0.1, + "ars": 12.0, + "inc": 88.0, + "ecc": 0.0, + "omega": 0.0, + } + + modeled = detrend_flux_on_out_of_transit_baseline(times, flux, unc, fit, min_side_points=2) + fallback = detrend_flux_on_out_of_transit_baseline(times, flux, unc, fit, prior=prior, min_side_points=2) + prior_coverage = summarize_prior_transit_coverage(times, prior, flux_values=flux, flux_errors=unc) + + assert modeled["applied"] is False + assert prior_coverage["valid"] is True + assert prior_coverage["has_two_sided_oot"] is True + assert fallback["applied"] is True + assert fallback["used_prior_ephemeris"] is True + assert "ephemeris-centered transit window" in fallback["note"] + + +def test_detrend_flux_on_out_of_transit_baseline_requires_default_side_coverage(): + times = np.linspace(-0.08, 0.08, 17) + baseline = 1.0 + 0.25 * times + transit_profile = np.ones_like(times) + transit_profile[(times >= -0.01) & (times <= 0.01)] = 0.99 + flux = baseline * transit_profile + unc = np.full_like(times, 0.01) + fit = types.SimpleNamespace( + transit=transit_profile, + parameters={"tmid": 0.0}, + ) + + result = detrend_flux_on_out_of_transit_baseline(times, flux, unc, fit) + + assert result["applied"] is False + assert result["pre_points"] <= 12 + assert result["post_points"] <= 12 + assert "need more than 12 on each side" in result["note"] + + +def test_deduplicate_comparison_star_coords_preserves_nearby_user_stars(): + preserved_coords, duplicate_messages = deduplicate_comparison_star_coords( + [ + [1826.0, 1499.0], + [1827.0, 1511.0], + [1828.0, 1487.0], + [842.0, 1810.0], + ], + min_separation_pixels=15.0, + ) + + assert preserved_coords == [ + [1826.0, 1499.0], + [1827.0, 1511.0], + [1828.0, 1487.0], + [842.0, 1810.0], + ] + assert duplicate_messages == [] + + +def test_robust_flux_floor_mask_rejects_tiny_positive_outliers(): + flux = np.full(30, 100.0) + flux[[5, 17]] = [1.0, 0.5] + + mask = robust_flux_floor_mask(flux) + + assert mask.sum() == 28 + assert not mask[5] + assert not mask[17] + + +def test_robust_target_reference_flux_mask_requires_both_series_to_be_plausible(): + target_flux = np.full(30, 100.0) + reference_flux = np.full(30, 120.0) + target_flux[7] = 1.0 + reference_flux[13] = 1.0 + + mask = robust_target_reference_flux_mask(target_flux, reference_flux) + + assert mask.sum() == 28 + assert not mask[7] + assert not mask[13] + + +def test_psf_frame_quality_mask_rejects_high_seeing_and_low_amplitude_outliers(): + frame_count = 30 + phase = np.linspace(0.0, 2.0 * np.pi, frame_count) + psf_rows = np.zeros((frame_count, 7), dtype=float) + psf_rows[:, 0] = 10.0 + psf_rows[:, 1] = 20.0 + psf_rows[:, 2] = 200.0 * (1.0 + 0.02 * np.sin(phase)) + psf_rows[:, 3] = 1.1 * (1.0 + 0.01 * np.cos(phase)) + psf_rows[:, 4] = 1.0 * (1.0 + 0.01 * np.sin(phase)) + psf_rows[5, 2] = 45.0 + psf_rows[12, 3:5] = 6.5 + + components = psf_frame_quality_components(psf_rows) + mask = psf_frame_quality_mask(psf_rows) + + assert mask.sum() == frame_count - 2 + assert not mask[5] + assert not mask[12] + assert components["amplitude_outlier_mask"][5] + assert components["seeing_outlier_mask"][12] + + +def test_target_psf_shape_quality_rejects_broad_target_but_preserves_amplitude_dips(): + frame_count = 30 + phase = np.linspace(0.0, 2.0 * np.pi, frame_count) + target_rows = np.zeros((frame_count, 7), dtype=float) + target_rows[:, 0] = 10.0 + target_rows[:, 1] = 20.0 + target_rows[:, 2] = 200.0 * (1.0 + 0.02 * np.sin(phase)) + target_rows[:, 3] = 1.0 + target_rows[:, 4] = 1.0 + target_rows[5, 2] = 45.0 + target_rows[12, 3:5] = 6.5 + + reference_rows = target_rows.copy() + reference_rows[:, 2] = 250.0 + reference_rows[:, 3:5] = 1.0 + + components = target_psf_shape_quality_components(target_rows, reference_rows) + mask = target_psf_shape_quality_mask(target_rows, reference_rows) + + assert mask.sum() == frame_count - 1 + assert mask[5] + assert not mask[12] + assert not components["invalid_mask"][5] + assert components["seeing_outlier_mask"][12] + assert components["reference_width_outlier_mask"][12] + + +def test_build_target_fit_candidate_jobs_masks_pairwise_psf_failures_but_preserves_target_dips(): + frame_count = 30 + + def build_psf_rows(amplitudes): + psf_rows = np.zeros((frame_count, 7), dtype=float) + psf_rows[:, 0] = 10.0 + psf_rows[:, 1] = 20.0 + psf_rows[:, 2] = amplitudes + psf_rows[:, 3] = 1.0 + psf_rows[:, 4] = 1.0 + return psf_rows + + target_amplitudes = np.full(frame_count, 100.0) + comp_amplitudes = np.full(frame_count, 120.0) + target_amplitudes[7] = 80.0 + comp_amplitudes[13] = 1.0 + + psf_data = { + "target": build_psf_rows(target_amplitudes), + "comp1": build_psf_rows(comp_amplitudes), + } + psf_data["target"][19, 3:5] = 6.5 + + candidate_jobs = build_target_fit_candidate_jobs( + psf_data, + aper_data=None, + apers=None, + annuli=None, + airmass=np.linspace(1.0, 1.3, frame_count), + comp_stars=[[1827.0, 1511.0]], + sigma=3.0, + require_comp_star=True, + skip_low_comparison_coverage_rejection=False, + use_psf_photometry=True, + use_aperture_photometry=False, + ) + + assert len(candidate_jobs) == 1 + assert candidate_jobs[0]["method"] == "psf" + assert candidate_jobs[0]["mask"].sum() == 28 + assert candidate_jobs[0]["mask"][7] + assert not candidate_jobs[0]["mask"][13] + assert not candidate_jobs[0]["mask"][19] + assert candidate_jobs[0]["coverage_count"] == 29 + + +def test_build_target_fit_candidate_jobs_uses_psf_flux_rows_for_psf_quality(): + frame_count = 30 + + def build_psf_rows(amplitudes): + psf_rows = np.zeros((frame_count, 7), dtype=float) + psf_rows[:, 0] = 10.0 + psf_rows[:, 1] = 20.0 + psf_rows[:, 2] = amplitudes + psf_rows[:, 3] = 1.0 + psf_rows[:, 4] = 1.0 + return psf_rows + + psf_data = { + "target": build_psf_rows(np.full(frame_count, 100.0)), + "comp1": build_psf_rows(np.full(frame_count, 120.0)), + } + psf_data["target"][19, 3:5] = 6.5 + + psf_flux_data = { + "target": build_psf_rows(np.full(frame_count, 100.0)), + "comp1": build_psf_rows(np.full(frame_count, 120.0)), + } + psf_flux_data["target"][7, 2] = 80.0 + psf_flux_data["target"][21, 3:5] = 6.5 + psf_flux_data["comp1"][13, 2] = 1.0 + + candidate_jobs = build_target_fit_candidate_jobs( + psf_data, + aper_data=None, + apers=None, + annuli=None, + airmass=np.linspace(1.0, 1.3, frame_count), + comp_stars=[[1827.0, 1511.0]], + sigma=3.0, + require_comp_star=True, + skip_low_comparison_coverage_rejection=False, + use_psf_photometry=True, + use_aperture_photometry=False, + psf_flux_data=psf_flux_data, + ) + + assert len(candidate_jobs) == 1 + assert candidate_jobs[0]["method"] == "psf" + assert candidate_jobs[0]["mask"].sum() == 28 + assert candidate_jobs[0]["mask"][7] + assert candidate_jobs[0]["mask"][19] + assert not candidate_jobs[0]["mask"][13] + assert not candidate_jobs[0]["mask"][21] + assert candidate_jobs[0]["coverage_count"] == 29 + + +@pytest.mark.filterwarnings("ignore::RuntimeWarning") +def test_legacy_psf_photometry_flux_row_uses_weighted_centroid_override(): + import exotic.exotic as exotic_module + + y_grid, x_grid = np.mgrid[0:31, 0:31] + data = exotic_module.gaussian_psf( + x_grid, + y_grid, + 15.25, + 14.65, + 200.0, + 1.8, + 2.2, + 0.05, + 30.0, + ) + seed_row = np.array([15.0, 15.0, 100.0, 1.0, 1.0, 0.0, 30.0], dtype=float) + + row = exotic_module.fit_legacy_psf_photometry_flux_row(data, seed_row, 0, box=8) + + xv, yv = exotic_module.mesh_box(seed_row[:2], 8, maxx=data.shape[1], maxy=data.shape[0]) + subarray = data[yv, xv] + expected_wx = np.sum(xv[0] * subarray.sum(0)) / subarray.sum(0).sum() + expected_wy = np.sum(yv[:, 0] * subarray.sum(1)) / subarray.sum(1).sum() + + assert row[0] == pytest.approx(expected_wx) + assert row[1] == pytest.approx(expected_wy) + assert row[2] > 0 + assert row[3] > 0 + assert row[4] > 0 + + +def test_load_psf_flux_seed_tracks_accepts_legacy_selected_comp_file(tmp_path): + import exotic.exotic as exotic_module + + run_dir = tmp_path / "old_run" + temp_dir = run_dir / "temp" + temp_dir.mkdir(parents=True) + target_rows = np.tile(np.array([[10.0, 20.0, 100.0, 1.0, 1.1, 0.0, 30.0]]), (3, 1)) + comp_rows = np.tile(np.array([[30.0, 40.0, 150.0, 1.2, 1.3, 0.0, 31.0]]), (3, 1)) + np.savetxt(temp_dir / "psf_data_target.txt", target_rows) + np.savetxt(temp_dir / "psf_data_comp.txt", comp_rows) + + seed_tracks = exotic_module.load_psf_flux_seed_tracks(str(run_dir), 3, ["comp1"]) + + assert set(seed_tracks) == {"target", "comp1"} + assert seed_tracks["target"].shape == (3, 7) + assert seed_tracks["comp1"].shape == (3, 7) + assert seed_tracks["target"][0, 0] == pytest.approx(10.0) + assert seed_tracks["comp1"][0, 0] == pytest.approx(30.0) + + +def test_centroid_offset_matches_reference_uses_float_geometry_tolerance(): + target = np.array([2383.27, 867.04, 6.3, 9.0, 0.7, 0.0, 223.0]) + comp = np.array([1821.90, 549.21, 131.0, 3.0, 4.9, 0.0, 226.0]) + + assert centroid_offset_matches_reference(comp, target, 562.88, 315.42) + + +def test_should_keep_header_wcs_alignment_prefers_geometry_over_flux_swings(): + decision = should_keep_header_wcs_alignment( + projected_off_frame=False, + frame_index=5, + target_psf_row=np.array([2383.27, 867.04, 6.3, 9.0, 0.7, 0.0, 223.0]), + previous_target_psf_row=np.array([2382.88, 454.14, 154.0, 2.8, 2.7, 0.0, 223.0]), + comp_psf_rows={ + "comp1": np.array([1821.90, 549.21, 131.0, 3.0, 4.9, 0.0, 226.0]), + }, + previous_comp_psf_rows={ + "comp1": np.array([1822.57, 135.19, 3645.8, 2.9, 5.8, 0.0, 226.0]), + }, + expected_offsets={ + "comp1": np.array([562.88, 315.42]), + }, + ) + + assert decision["use_wcs_alignment"] is True + assert decision["reason"] == "geometry_match" + assert not decision["target_flux_change_ok"] + assert not decision["comp_flux_change_ok"] + assert decision["geometry_match_count"] == 1 + + +def test_should_keep_header_wcs_alignment_ignores_one_bad_comp_when_majority_match(): + decision = should_keep_header_wcs_alignment( + projected_off_frame=False, + frame_index=8, + target_psf_row=np.array([2387.50, 868.60, 50.0, 3.0, 3.0, 0.0, 223.0]), + previous_target_psf_row=np.array([2382.88, 454.14, 154.0, 2.8, 2.7, 0.0, 223.0]), + comp_psf_rows={ + "comp1": np.array([1827.0, 558.1, 200.0, 3.0, 5.0, 0.0, 226.0]), + "comp2": np.array([1300.0, 900.0, 300.0, 3.0, 5.0, 0.0, 226.0]), + }, + previous_comp_psf_rows={ + "comp1": np.array([1822.6, 135.2, 3645.8, 2.9, 5.8, 0.0, 226.0]), + "comp2": np.array([1600.0, 700.0, 280.0, 3.0, 5.0, 0.0, 226.0]), + }, + expected_offsets={ + "comp1": np.array([560.7, 310.2]), + "comp2": np.array([1187.5, 31.4]), + }, + ) + + assert decision["use_wcs_alignment"] is True + assert decision["geometry_test_count"] == 2 + assert decision["geometry_match_count"] == 1 + + +def test_should_keep_header_wcs_alignment_rejects_geometry_mismatch(): + decision = should_keep_header_wcs_alignment( + projected_off_frame=False, + frame_index=8, + target_psf_row=np.array([1576.0, 1317.0, 1.1, 20.0, 1.9, 0.0, 223.0]), + previous_target_psf_row=np.array([2382.88, 454.14, 154.0, 2.8, 2.7, 0.0, 223.0]), + comp_psf_rows={ + "comp1": np.array([1826.7, 1510.9, 1.9, 15.4, 1.5, 0.0, 226.0]), + }, + previous_comp_psf_rows={ + "comp1": np.array([1822.57, 135.19, 3645.8, 2.9, 5.8, 0.0, 226.0]), + }, + expected_offsets={ + "comp1": np.array([562.88, 315.42]), + }, + ) + + assert decision["use_wcs_alignment"] is False + assert decision["reason"] == "geometry_mismatch" + assert decision["geometry_match_count"] == 0 + + +def test_check_coordinates_non_interactive_prefers_wcs_centroid(): + x_pixel, y_pixel = check_coordinates( + input_x_pixel=5, + input_y_pixel=5, + centroid_x=100.25, + centroid_y=200.75, + sigma_x=1.0, + sigma_y=1.0, + calculated_x_pixel=100, + calculated_y_pixel=201, + non_interactive_run=True, + ) + + assert x_pixel == 100.25 + assert y_pixel == 200.75 + + +def test_check_coordinates_non_interactive_keeps_input_when_wcs_psf_is_implausible(): + x_pixel, y_pixel = check_coordinates( + input_x_pixel=246, + input_y_pixel=271, + centroid_x=238.5, + centroid_y=266.1, + sigma_x=4.6, + sigma_y=0.7, + calculated_x_pixel=245, + calculated_y_pixel=270, + non_interactive_run=True, + wcs_psf_quality_score=np.inf, + input_psf_quality_score=0.1, + ) + + assert x_pixel == 246 + assert y_pixel == 271 + + +def test_check_coordinates_non_interactive_keeps_plausible_input_when_wcs_finds_other_source(): + x_pixel, y_pixel = check_coordinates( + input_x_pixel=246, + input_y_pixel=271, + centroid_x=236.6, + centroid_y=265.2, + sigma_x=1.2, + sigma_y=0.8, + calculated_x_pixel=236, + calculated_y_pixel=265, + non_interactive_run=True, + wcs_psf_quality_score=0.05, + input_psf_quality_score=0.25, + ) + + assert x_pixel == 246 + assert y_pixel == 271 + + +def test_check_coordinates_non_interactive_uses_wcs_pixel_when_centroid_is_nan(): + x_pixel, y_pixel = check_coordinates( + input_x_pixel=5, + input_y_pixel=5, + centroid_x=float("nan"), + centroid_y=float("nan"), + sigma_x=1.0, + sigma_y=1.0, + calculated_x_pixel=100, + calculated_y_pixel=201, + non_interactive_run=True, + ) + + assert x_pixel == 100 + assert y_pixel == 201 + + +def test_check_coordinates_can_prefer_input_pixels_over_wcs_conflict(): + x_pixel, y_pixel = check_coordinates( + input_x_pixel=5, + input_y_pixel=5, + centroid_x=100.25, + centroid_y=200.75, + sigma_x=1.0, + sigma_y=1.0, + calculated_x_pixel=100, + calculated_y_pixel=201, + non_interactive_run=True, + prefer_pixel_values_over_wcs_for_target="y", + ) + + assert x_pixel == 5 + assert y_pixel == 5 + + +def test_should_prefer_pixel_values_over_wcs_for_target_parses_values(): + assert should_prefer_pixel_values_over_wcs_for_target(None) is False + assert should_prefer_pixel_values_over_wcs_for_target("n") is False + assert should_prefer_pixel_values_over_wcs_for_target("y") is True + assert should_prefer_pixel_values_over_wcs_for_target(True) is True + + +def test_psf_solution_quality_score_rejects_offset_or_elongated_solutions(): + good = np.array([246.2, 270.8, 140.0, 1.1, 0.9, 0.0, 40.0]) + offset = np.array([238.5, 266.1, 140.0, 1.1, 0.9, 0.0, 40.0]) + elongated = np.array([246.2, 270.8, 140.0, 4.6, 0.7, 0.0, 40.0]) + + assert np.isfinite(psf_solution_quality_score(good, seed_pos=[246.0, 271.0])) + assert not np.isfinite(psf_solution_quality_score(offset, seed_pos=[246.0, 271.0])) + assert not np.isfinite(psf_solution_quality_score(elongated, seed_pos=[246.0, 271.0])) + + +def test_alignment_candidate_selection_rejects_broad_offset_wcs_target_solution(): + psf_data = { + "target": np.zeros((2, 7), dtype=float), + "comp1": np.zeros((2, 7), dtype=float), + } + psf_data["target"][0] = [245.8, 270.7, 110.0, 1.1, 0.8, 0.0, 40.0] + psf_data["comp1"][0] = [360.3, 443.3, 174.0, 1.1, 1.0, 0.0, 40.0] + tar_comp_dist = {"comp1": np.array([115.0, 173.0])} + + wcs_candidate = { + "coords": np.array([[244.0, 268.7], [360.4, 443.2]], dtype=float), + "projected_off_frame": False, + "psf_rows": { + "target": np.array([244.0, 268.7, 210.0, 7.3, 5.6, 0.0, 40.0]), + "comp1": np.array([360.4, 443.2, 174.0, 1.1, 1.0, 0.0, 40.0]), + }, + "warnings": [], + } + fallback_candidate = { + "coords": np.array([[246.0, 270.8], [360.2, 443.3]], dtype=float), + "psf_rows": { + "target": np.array([245.9, 270.8, 111.0, 1.1, 0.8, 0.0, 40.0]), + "comp1": np.array([360.2, 443.3, 173.0, 1.1, 1.0, 0.0, 40.0]), + }, + "warnings": [], + } + + assert not np.isfinite( + alignment_candidate_quality_score( + wcs_candidate, + comp_keys=["comp1"], + previous_target_psf_row=psf_data["target"][0], + previous_comp_psf_rows={"comp1": psf_data["comp1"][0]}, + expected_offsets=tar_comp_dist, + ) + ) + + selected_source, selected_candidate, diagnostics = select_alignment_candidate( + {"wcs": wcs_candidate, "fallback": fallback_candidate, "file_name": "frame.fits"}, + frame_index=1, + psf_data=psf_data, + tar_comp_dist=tar_comp_dist, + comp_keys=["comp1"], + ) + + assert selected_source == "fallback" + assert selected_candidate is fallback_candidate + assert not np.isfinite(diagnostics["wcs_score"]) + assert np.isfinite(diagnostics["fallback_score"]) + + +def test_is_comp_star_required_parses_values(): + assert is_comp_star_required(None) is True + assert is_comp_star_required("y") is True + assert is_comp_star_required("n") is False + + +def test_mixed_exposure_times_keep_target_only_allowed_with_scaling_warning(monkeypatch): + import exotic.exotic as exotic_module + + messages = [] + monkeypatch.setattr( + exotic_module, + "log_info", + lambda message, **kwargs: messages.append((message, kwargs)), + ) + + assert exotic_module.exposure_time_spread_fraction([60.0, 60.3, 60.5]) < 0.01 + assert not exotic_module.exposure_variation_requires_comp_star([60.0, 60.3, 60.5]) + assert exotic_module.resolve_require_comp_star_for_exposure_times("n", [60.0, 60.3, 60.5]) is False + + assert exotic_module.exposure_time_spread_fraction([60.0, 61.0]) > 0.01 + assert exotic_module.exposure_variation_requires_comp_star([60.0, 61.0]) + assert exotic_module.resolve_require_comp_star_for_exposure_times("n", [60.0, 61.0]) is False + assert exotic_module.resolve_require_comp_star_for_exposure_times("y", [60.0, 61.0]) is True + assert any("scale source counts to a common exposure time" in message for message, _ in messages) + + +def test_img_time_bjd_tdb_prefers_direct_mid_exposure_bjd(monkeypatch): + import exotic.exotic as exotic_module + + header = exotic_module.fits.Header() + header["BJD_TDB"] = 2461152.1287422837 + header["DATE-AVG"] = "2026-04-22T15:05:23.333333" + header["DATE-UTC"] = "2026-04-22T15:05:08.333333" + header["EXPTIME"] = 30.0 + + monkeypatch.setattr( + exotic_module, + "convert_jd_to_bjd", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("conversion should not run")), + ) + + assert exotic_module.img_time_bjd_tdb(header, {}, {}) == pytest.approx(2461152.1287422837) + + +def test_img_time_bjd_tdb_uses_nina_date_avg_before_start_time(monkeypatch): + import exotic.exotic as exotic_module + + header = exotic_module.fits.Header() + header["DATE-AVG"] = "2026-04-22T15:05:23.333333" + header["DATE-UTC"] = "2026-04-22T15:05:08.333333" + header["EXPTIME"] = 10.0 + + converted_inputs = [] + + def fake_convert_jd_to_bjd(values, _p_dict, _info_dict): + converted_inputs.extend(values) + return np.asarray(values, dtype=float) + 0.25 + + monkeypatch.setattr(exotic_module, "convert_jd_to_bjd", fake_convert_jd_to_bjd) + + expected_midpoint_jd = exotic_module.Time("2026-04-22T15:05:23.333333", scale="utc").jd + expected_start_plus_exposure_jd = ( + exotic_module.Time("2026-04-22T15:05:08.333333", scale="utc").jd + + 5.0 / 86400.0 + ) + + assert exotic_module.img_time_jd(header) == pytest.approx(expected_midpoint_jd) + assert exotic_module.img_time_bjd_tdb(header, {}, {}) == pytest.approx(expected_midpoint_jd + 0.25) + assert converted_inputs == pytest.approx([expected_midpoint_jd]) + assert converted_inputs[0] != pytest.approx(expected_start_plus_exposure_jd, abs=1e-9) + + +def test_is_target_driven_comp_selection_enabled_parses_values(): + assert is_target_driven_comp_selection_enabled(None) is False + assert is_target_driven_comp_selection_enabled("y") is True + assert is_target_driven_comp_selection_enabled("n") is False + + +def test_should_skip_low_comparison_coverage_rejection_parses_values(): + assert should_skip_low_comparison_coverage_rejection(None) is False + assert should_skip_low_comparison_coverage_rejection("y") is True + assert should_skip_low_comparison_coverage_rejection("n") is False + + +def test_should_fit_lightcurve_to_every_comparison_candidate_parses_values(): + assert should_fit_lightcurve_to_every_comparison_candidate(None) is False + assert should_fit_lightcurve_to_every_comparison_candidate("y") is True + assert should_fit_lightcurve_to_every_comparison_candidate("n") is False + + +def test_stellar_variability_ensemble_config_defaults_on_and_supports_opt_out(): + assert should_use_ensemble_photometry_for_stellar_variability(None) is True + assert should_use_ensemble_photometry_for_stellar_variability("y") is True + assert should_use_ensemble_photometry_for_stellar_variability("n") is False + assert should_use_ensemble_photometry_for_stellar_variability(False) is False + + +def test_apparent_and_exact_comparison_config_defaults_and_values(): + assert should_require_apparent_magnitudes(None) is True + assert should_require_apparent_magnitudes("n") is False + assert should_use_exactly_the_comps_provided(None) is False + assert should_use_exactly_the_comps_provided("y") is True + + +def test_project_comparison_radec_to_reference_pixels_and_reject_out_of_frame(): + wcs = WCS(naxis=2) + wcs.wcs.crpix = [50.0, 50.0] + wcs.wcs.cdelt = np.array([-0.001, 0.001]) + wcs.wcs.crval = [31.04125, 46.68972] + wcs.wcs.ctype = ["RA---TAN", "DEC--TAN"] + + projected = project_comparison_radec_to_pixels( + [[31.04125, 46.68972]], + wcs.to_header(), + (100, 100), + ) + + assert projected[0] == pytest.approx([49.0, 49.0]) + with pytest.raises(ValueError, match="projects outside the reference image"): + project_comparison_radec_to_pixels( + [[32.04125, 46.68972]], + wcs.to_header(), + (100, 100), + ) + + +def test_independent_ensemble_comparison_limits_default_to_five_and_have_no_upper_cap(): + assert parse_maximum_number_of_ensemble_comparisons_for_transit(None) == 5 + assert parse_maximum_number_of_ensemble_comparisons_for_transit(1) == 5 + assert parse_maximum_number_of_ensemble_comparisons_for_transit("250") == 250 + assert parse_maximum_number_of_ensemble_comparisons_for_stellar_variability(None) == 5 + assert parse_maximum_number_of_ensemble_comparisons_for_stellar_variability(2) == 2 + assert parse_maximum_number_of_ensemble_comparisons_for_stellar_variability("125") == 125 + + +def test_limited_ensemble_comparison_keys_uses_configured_maximum(): + ranked_summaries = [ + {'key': f'comp{index}'} + for index in range(1, 13) + ] + + assert limited_ensemble_comparison_keys(ranked_summaries, None) == [ + f'comp{index}' for index in range(1, 6) + ] + assert limited_ensemble_comparison_keys(ranked_summaries, 12) == [ + f'comp{index}' for index in range(1, 13) + ] + + +def test_transit_ensemble_fit_uses_configured_maximum(monkeypatch): + frame_count = 6 + quality_mask = np.ones(frame_count, dtype=bool) + ranked_summaries = [ + { + 'key': f'comp{index}', + 'comp_index': index - 1, + 'aggregate_score': index / 1000.0, + 'coverage_rejected': False, + 'suitability_outlier_rejected': False, + 'psf_quality_keep_mask': quality_mask, + } + for index in range(1, 13) + ] + comparison_calibration = { + 'method': 'aperture', + 'a': 0, + 'an': 0, + 'aper': 5.0, + 'annulus': 12.0, + 'comp_summaries': ranked_summaries, + } + aper_data = { + 'target': np.full((frame_count, 1, 1), 1000.0), + **{ + f'comp{index}': np.full((frame_count, 1, 1), 100.0 + index) + for index in range(1, 13) + }, + } + captured = {} + + def capture_active_keys(comp_flux_map, active_keys, validity_mask_func): + captured['active_keys'] = list(active_keys) + raise RuntimeError('captured configured transit ensemble') + + monkeypatch.setattr( + 'exotic.exotic.build_absolute_comp_ensemble_flux', + capture_active_keys, + ) + + with pytest.raises(RuntimeError, match='captured configured transit ensemble'): + fit_ranked_comparison_calibration_candidates( + np.linspace(0.0, 0.05, frame_count), + np.linspace(2460000.0, 2460000.05, frame_count), + np.linspace(1.0, 1.2, frame_count), + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.ones(frame_count), + use_ensemble_photometry_rather_than_single_comp=True, + maximum_number_of_ensemble_comparisons_for_transit=9, + ) + + assert captured['active_keys'] == [ + f'comp{index}' for index in range(1, 10) + ] + + +def test_exact_transit_ensemble_uses_every_supplied_comparison(monkeypatch): + frame_count = 6 + quality_mask = np.ones(frame_count, dtype=bool) + comparison_calibration = { + 'method': 'aperture', + 'a': 0, + 'an': 0, + 'aper': 5.0, + 'annulus': 12.0, + 'comp_summaries': [ + { + 'key': f'comp{index}', + 'comp_index': index - 1, + 'aggregate_score': index / 1000.0, + 'coverage_rejected': False, + 'suitability_outlier_rejected': False, + 'psf_quality_keep_mask': quality_mask, + } + for index in range(1, 6) + ], + } + aper_data = { + 'target': np.full((frame_count, 1, 1), 1000.0), + **{ + f'comp{index}': np.full((frame_count, 1, 1), 100.0 + index) + for index in range(1, 6) + }, + } + captured = {} + + def capture_active_keys(comp_flux_map, active_keys, validity_mask_func): + captured['active_keys'] = list(active_keys) + raise RuntimeError('captured exact transit ensemble') + + monkeypatch.setattr( + 'exotic.exotic.build_absolute_comp_ensemble_flux', + capture_active_keys, + ) + + with pytest.raises(RuntimeError, match='captured exact transit ensemble'): + fit_ranked_comparison_calibration_candidates( + np.linspace(0.0, 0.05, frame_count), + np.linspace(2460000.0, 2460000.05, frame_count), + np.linspace(1.0, 1.2, frame_count), + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.ones(frame_count), + use_ensemble_photometry_rather_than_single_comp=True, + maximum_number_of_ensemble_comparisons_for_transit=1, + use_exactly_the_comps_provided=True, + ) + + assert captured['active_keys'] == [f'comp{index}' for index in range(1, 6)] + + +def test_exact_comparison_mode_does_not_expand_tracked_pool(): + supplied = [[10.0, 20.0], [30.0, 40.0]] + automatic = [[50.0, 60.0], [70.0, 80.0]] + + tracked, messages = build_tracked_comparison_pool( + supplied, + automatic, + use_exactly_the_comps_provided=True, + ) + + assert tracked == supplied + assert messages == [] + + +def test_fortuitous_variable_photometry_config_defaults_on_and_supports_opt_out(): + assert should_photometer_fortuitous_variables(None) is True + assert should_photometer_fortuitous_variables("y") is True + assert should_photometer_fortuitous_variables("n") is False + assert should_photometer_fortuitous_variables(False) is False + + +def test_should_detect_bad_pixels_before_photometry_parses_values(): + assert should_detect_bad_pixels_before_photometry(None) is False + assert should_detect_bad_pixels_before_photometry("y") is True + assert should_detect_bad_pixels_before_photometry("n") is False + + +def test_get_multiprocess_bad_pixel_precheck_processes_parses_values(): + assert get_multiprocess_bad_pixel_precheck_processes(None) is None + assert get_multiprocess_bad_pixel_precheck_processes("n") is None + assert get_multiprocess_bad_pixel_precheck_processes("0") is None + assert get_multiprocess_bad_pixel_precheck_processes("y") >= 1 + assert get_multiprocess_bad_pixel_precheck_processes("3") == 3 + assert get_multiprocess_bad_pixel_precheck_processes(2) == 2 + + +def test_is_out_of_transit_baseline_detrending_enabled_parses_values(): + assert is_out_of_transit_baseline_detrending_enabled(None) is True + assert is_out_of_transit_baseline_detrending_enabled("y") is True + assert is_out_of_transit_baseline_detrending_enabled("n") is False + assert is_out_of_transit_baseline_detrending_enabled(True) is True + + +def test_get_final_fit_baseline_duration_multiplier_parses_values(): + assert get_final_fit_baseline_duration_multiplier(None) == pytest.approx(1.0) + assert get_final_fit_baseline_duration_multiplier("2.5") == pytest.approx(2.5) + assert get_final_fit_baseline_duration_multiplier(0) == pytest.approx(0.0) + assert get_final_fit_baseline_duration_multiplier(-1) == pytest.approx(1.0) + + +def test_should_use_psf_photometry_parses_values(): + assert should_use_psf_photometry(None) is True + assert should_use_psf_photometry("y") is True + assert should_use_psf_photometry("n") is False + + +def test_should_use_aperture_photometry_parses_values(): + assert should_use_aperture_photometry(None) is True + assert should_use_aperture_photometry("y") is True + assert should_use_aperture_photometry("n") is False + + +def test_should_use_aperture_corrections_and_full_image_fwhm_parses_values(): + assert should_use_aperture_corrections_and_full_image_fwhm(None) is False + assert should_use_aperture_corrections_and_full_image_fwhm("y") is True + assert should_use_aperture_corrections_and_full_image_fwhm("n") is False + assert should_use_aperture_corrections_and_full_image_fwhm(True) is True + + +def test_overexposure_rejection_config_parsers_default_and_override(): + assert should_reject_overexposed_stars(None) is True + assert should_reject_overexposed_stars("y") is True + assert should_reject_overexposed_stars("n") is False + assert should_reject_overexposed_stars(False) is False + + assert parse_saturation_value(None) == pytest.approx(65535.0) + assert parse_saturation_value("") == pytest.approx(65535.0) + assert parse_saturation_value("42000") == pytest.approx(42000.0) + assert parse_saturation_value(-1) == pytest.approx(65535.0) + assert parse_saturation_value("not-a-number") == pytest.approx(65535.0) + + assert parse_overexposure_threshold_fraction(None) == pytest.approx(0.9) + assert parse_overexposure_threshold_fraction("0.75") == pytest.approx(0.75) + assert parse_overexposure_threshold_fraction(1.0) == pytest.approx(1.0) + assert parse_overexposure_threshold_fraction(0) == pytest.approx(0.9) + assert parse_overexposure_threshold_fraction(1.5) == pytest.approx(0.9) + + +def test_saturation_value_from_header_uses_cecilia_microobservatory_value(): + assert saturation_value_from_header({"TELESCOP": "Cecilia "}) == pytest.approx(4096.0) + assert saturation_value_from_header({ + "TELESCOP": "Cecilia ", + "SATURATE": 65535.0, + }) == pytest.approx(4096.0) + assert saturation_value_from_header({"SATURATE": 76500.0}) == pytest.approx(76500.0) + assert saturation_value_from_header({}) is None + + +def test_should_use_eebls_to_initialize_tmid_and_bounds_parses_values(): + assert should_use_eebls_to_initialize_tmid_and_bounds(None) is True + assert should_use_eebls_to_initialize_tmid_and_bounds("y") is True + assert should_use_eebls_to_initialize_tmid_and_bounds("n") is False + assert should_use_eebls_to_initialize_tmid_and_bounds(True) is True + + +def test_should_pick_comparison_by_eebls_snr_parses_values(): + assert should_pick_comparison_by_eebls_snr(None) is True + assert should_pick_comparison_by_eebls_snr("y") is True + assert should_pick_comparison_by_eebls_snr("n") is False + assert should_pick_comparison_by_eebls_snr(True) is True + + +def test_should_use_deviation_from_expected_transit_in_qc_parses_values(): + assert should_use_deviation_from_expected_transit_in_qc(None) is True + assert should_use_deviation_from_expected_transit_in_qc("y") is True + assert should_use_deviation_from_expected_transit_in_qc("n") is False + assert should_use_deviation_from_expected_transit_in_qc(True) is True + + +def test_parse_deviation_from_expected_transit_in_qc_sigma_parses_values(): + assert parse_deviation_from_expected_transit_in_qc_sigma(None) == pytest.approx(5.0) + assert parse_deviation_from_expected_transit_in_qc_sigma("7.5") == pytest.approx(7.5) + assert parse_deviation_from_expected_transit_in_qc_sigma(3) == pytest.approx(3.0) + assert parse_deviation_from_expected_transit_in_qc_sigma(-1) == pytest.approx(5.0) + + +def test_should_exit_at_first_qc_pass_solution_parses_values(): + assert should_exit_at_first_qc_pass_solution(None) is True + assert should_exit_at_first_qc_pass_solution("y") is True + assert should_exit_at_first_qc_pass_solution("n") is False + assert should_exit_at_first_qc_pass_solution(True) is True + + +def test_validate_ultranest_mpi_runtime_rejects_whole_program_mpi(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr( + exotic_module, + "get_mpi_status", + lambda: {"available": True, "size": 72, "rank": 0, "source": "mpi4py", "error": None}, + ) + + with pytest.raises(RuntimeError, match="duplicates the full reduction"): + exotic_module.validate_ultranest_mpi_runtime() + + +def test_configure_windows_multiprocessing_main_spec_retargets_console_launcher(monkeypatch): + import exotic.exotic as exotic_module + + fake_main = types.SimpleNamespace( + __spec__=types.SimpleNamespace(name="exotic"), + __file__=r"C:\Python312\Scripts\exotic.exe", + __package__="", + ) + spawn_executables = [] + + monkeypatch.setattr(exotic_module.sys, "platform", "win32") + monkeypatch.setattr(exotic_module.sys, "_base_executable", r"C:\Python312\python.exe", raising=False) + monkeypatch.setattr(exotic_module.sys, "executable", r"C:\Python312\Scripts\exotic.exe") + monkeypatch.setattr(exotic_module.sys, "frozen", True, raising=False) + monkeypatch.setattr(exotic_module.multiprocessing, "set_executable", spawn_executables.append) + monkeypatch.setitem(sys.modules, "__main__", fake_main) + + assert configure_windows_multiprocessing_main_spec() is True + assert fake_main.__spec__ is None + assert fake_main.__file__ is None + assert fake_main.__package__ is None + assert spawn_executables == [r"C:\Python312\python.exe"] + assert exotic_module.sys.frozen is False + + +def test_configure_windows_multiprocessing_main_spec_skips_non_windows(monkeypatch): + import exotic.exotic as exotic_module + + fake_main = types.SimpleNamespace(__spec__=types.SimpleNamespace(name="exotic"), __file__="exotic.exe") + + monkeypatch.setattr(exotic_module.sys, "platform", "linux") + monkeypatch.setitem(sys.modules, "__main__", fake_main) + + assert configure_windows_multiprocessing_main_spec() is False + assert fake_main.__spec__.name == "exotic" + + +def test_configure_windows_multiprocessing_main_spec_preserves_regular_script(monkeypatch): + import exotic.exotic as exotic_module + + fake_spec = types.SimpleNamespace(name="run_exotic") + fake_main = types.SimpleNamespace( + __spec__=fake_spec, + __file__=r"C:\work\run_exotic.py", + __package__="", + ) + + monkeypatch.setattr(exotic_module.sys, "platform", "win32") + monkeypatch.setattr(exotic_module.sys, "_base_executable", r"C:\Python312\python.exe", raising=False) + monkeypatch.setattr(exotic_module.sys, "executable", r"C:\Python312\python.exe") + monkeypatch.setattr(exotic_module.multiprocessing, "set_executable", lambda _path: None) + monkeypatch.setitem(sys.modules, "__main__", fake_main) + + assert configure_windows_multiprocessing_main_spec() is True + assert fake_main.__spec__ is fake_spec + assert fake_main.__file__ == r"C:\work\run_exotic.py" + + +def test_windows_python_spawn_executable_falls_back_to_exec_prefix(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module.sys, "_base_executable", r"C:\Python312\Scripts\exotic.exe", raising=False) + monkeypatch.setattr(exotic_module.sys, "executable", r"C:\Python312\Scripts\exotic.exe") + monkeypatch.setattr(exotic_module.sys, "exec_prefix", r"C:\Python312") + monkeypatch.setattr(exotic_module.sys, "base_exec_prefix", r"C:\Python312", raising=False) + + assert exotic_module._windows_python_spawn_executable() == r"C:\Python312\python.exe" + + +def test_process_pool_executor_uses_threads_on_windows(monkeypatch): + import exotic.exotic as exotic_module + + captured = {} + + class FakeThreadPoolExecutor: + def __init__(self, *args, **kwargs): + captured["args"] = args + captured["kwargs"] = kwargs + + monkeypatch.setattr(exotic_module.sys, "platform", "win32") + monkeypatch.setattr(exotic_module, "ThreadPoolExecutor", FakeThreadPoolExecutor) + + executor = exotic_module.ProcessPoolExecutor(max_workers=3, initializer=lambda: None) + + assert isinstance(executor, FakeThreadPoolExecutor) + assert captured["kwargs"]["max_workers"] == 3 + assert "initializer" in captured["kwargs"] + + +def test_build_time_rejection_diagnostic_groups_contiguous_ranges(): + times = np.array([1.0, 1.1, 1.2, 1.5, 1.6, 2.0], dtype=float) + keep_mask = np.array([True, False, False, True, False, True], dtype=bool) + + diagnostic = build_time_rejection_diagnostic("Example stage", times, keep_mask) + + assert diagnostic["stage"] == "Example stage" + assert diagnostic["input_point_count"] == 6 + assert diagnostic["kept_point_count"] == 3 + assert diagnostic["dropped_point_count"] == 3 + assert diagnostic["dropped_ranges"] == [ + {"start": pytest.approx(1.1), "end": pytest.approx(1.2), "count": 2}, + {"start": pytest.approx(1.6), "end": pytest.approx(1.6), "count": 1}, + ] + + +def test_estimate_tmid_and_bounds_with_eebls_identifies_box_like_transit(): + times = np.linspace(0.0, 0.2, 240) + tmid = 0.101 + duration = 0.028 + flux = np.ones(times.shape[0], dtype=float) + in_transit = np.abs(times - tmid) <= duration / 2.0 + flux[in_transit] -= 0.018 + flux += 0.0015 * (times - np.nanmean(times)) + flux_errors = np.full(times.shape[0], 0.002, dtype=float) + prior = { + "tmid": 0.08, + "per": 1.0, + "rprs": np.sqrt(0.018), + "ars": 12.0, + "inc": 88.5, + "ecc": 0.0, + "omega": 0.0, + } + + summary = estimate_tmid_and_bounds_with_eebls( + times, + flux, + flux_errors, + prior, + [0.04, 0.12], + ) + + assert summary["applied"] is True + assert summary["method"] == "eebls" + assert summary["tmid"] == pytest.approx(tmid, abs=0.01) + assert summary["bounds"][0] < summary["tmid"] < summary["bounds"][1] + assert summary["depth"] > 0 + assert summary["depth_snr"] > 0 + + +def test_estimate_tmid_and_bounds_with_eebls_keeps_depth_snr_for_one_sided_event(): + times = np.linspace(0.0, 0.11, 160) + tmid = 0.101 + duration = 0.028 + flux = np.ones(times.shape[0], dtype=float) + in_transit = np.abs(times - tmid) <= duration / 2.0 + flux[in_transit] -= 0.018 + flux_errors = np.full(times.shape[0], 0.002, dtype=float) + prior = { + "tmid": 0.08, + "per": 1.0, + "rprs": np.sqrt(0.018), + "ars": 12.0, + "inc": 88.5, + "ecc": 0.0, + "omega": 0.0, + } + + summary = estimate_tmid_and_bounds_with_eebls( + times, + flux, + flux_errors, + prior, + [0.04, 0.12], + ) + + assert summary["method"] == "eebls" + assert summary["applied"] is False + assert summary["depth"] > 0 + assert summary["depth_snr"] > 0 + assert "keeping the EEBLS depth SNR only" in summary["note"] + + +def test_estimate_ephemeris_tmid_and_bounds_caps_bracketed_runs_to_duration_scale(): + prior = { + "tmid": 2458247.90746, + "per": 1.645321, + "rprs": 0.1265, + "ars": 4.9, + "inc": 85.87, + "ecc": 0.0, + "omega": 0.0, + } + times = np.linspace(2461151.8012, 2461151.9966, 220) + expected_duration = 0.049 + + summary = estimate_ephemeris_tmid_and_bounds( + times, + prior["tmid"], + prior["per"], + midt_unc=0.00036, + per_unc=1.0e-5, + expected_duration=expected_duration, + sigma_multiplier=35.0, + ) + + assert summary["duration_capped"] is True + assert summary["observed_window_capped"] is True + assert summary["observations_bracket_expected_transit"] is True + assert summary["tmid"] == pytest.approx(2461151.899025, abs=1e-6) + assert summary["half_width"] > 0 + assert summary["bounds"][0] == pytest.approx(times.min() + 0.5 * expected_duration) + assert summary["bounds"][1] == pytest.approx(times.max() - 0.5 * expected_duration) + + +def test_estimate_ephemeris_tmid_and_bounds_keeps_wider_bounds_for_one_sided_runs(): + prior = { + "tmid": 2458247.90746, + "per": 1.645321, + } + times = np.linspace(2461151.92, 2461152.02, 120) + + summary = estimate_ephemeris_tmid_and_bounds( + times, + prior["tmid"], + prior["per"], + midt_unc=0.00036, + per_unc=1.0e-5, + expected_duration=0.049, + sigma_multiplier=35.0, + ) + + assert summary["duration_capped"] is False + assert summary["observations_bracket_expected_transit"] is False + assert summary["half_width"] == pytest.approx(0.25 * prior["per"]) + + +def test_estimate_ephemeris_tmid_and_bounds_selects_nearest_epoch_for_ingress_only_runs(): + # Regression for issue #1387: an ingress-only partial transit whose true mid + # falls minutes AFTER the last surviving frame. floor(phases).max() snapped to + # the previous cycle and reported Tmid one full period early; the epoch nearest + # the data is the correct one. Geometry taken from the 2026-07-27 TOI-1516 b + # MicroObservatory night that surfaced the bug (two independent reductions + # reported 2461246.93, one period before the actual night of the frames). + prior_tmid = 2458765.325 + period = 2.056014 + times = np.linspace(2461248.8938, 2461248.9847, 34) + expected_mid = prior_tmid + 1208 * period # 2461248.9899, ~7 min after times.max() + + summary = estimate_ephemeris_tmid_and_bounds( + times, + prior_tmid, + period, + midt_unc=0.00023, + per_unc=2.1e-6, + expected_duration=0.1177, + sigma_multiplier=25.0, + ) + + assert summary["cycle_index"] == pytest.approx(1208.0) + assert summary["tmid"] == pytest.approx(expected_mid, abs=1e-6) + # The search bounds must be able to reach the true mid. + assert summary["bounds"][0] <= expected_mid <= summary["bounds"][1] + assert summary["observations_bracket_expected_transit"] is False + + +def test_is_adaptive_aperture_mode_enabled_parses_values(): + assert is_adaptive_aperture_mode_enabled(None) is False + assert is_adaptive_aperture_mode_enabled("y") is True + assert is_adaptive_aperture_mode_enabled("n") is False + assert is_adaptive_aperture_mode_enabled(True) is True + + +def test_aperture_sigma_bounds_match_physical_fwhm_limits(): + assert APERTURE_SIGMA_MIN == pytest.approx( + APERTURE_MIN_FWHM_MULTIPLIER * GAUSSIAN_SIGMA_TO_FWHM + ) + assert APERTURE_SIGMA_MAX == pytest.approx( + APERTURE_MAX_FWHM_MULTIPLIER * GAUSSIAN_SIGMA_TO_FWHM + ) + + +def test_aperture_correction_profile_recovers_gaussian_curve_of_growth(): + sigma = 2.0 + fwhm = GAUSSIAN_SIGMA_TO_FWHM * sigma + y, x = np.mgrid[0:120, 0:120] + image = np.full((120, 120), 10.0, dtype=float) + positions = np.array([ + [25.0, 25.0], + [25.0, 70.0], + [70.0, 25.0], + [70.0, 70.0], + [95.0, 95.0], + ]) + for xc, yc in positions: + image += 1200.0 * np.exp(-((x - xc) ** 2 + (y - yc) ** 2) / (2.0 * sigma ** 2)) + + field_star_psfs = np.column_stack([ + positions[:, 0], + positions[:, 1], + np.full(positions.shape[0], 1200.0), + np.full(positions.shape[0], sigma), + np.full(positions.shape[0], sigma), + np.zeros(positions.shape[0]), + np.full(positions.shape[0], 10.0), + ]) + radii = np.array([ + APERTURE_MIN_FWHM_MULTIPLIER * fwhm, + fwhm, + APERTURE_MAX_FWHM_MULTIPLIER * fwhm, + ]) + + profile = build_aperture_correction_profile( + image, + radii, + fwhm_hint=fwhm, + field_star_psfs=field_star_psfs, + ) + + assert profile["applied"] is True + assert profile["star_count"] == positions.shape[0] + assert profile["image_fwhm"] == pytest.approx(fwhm) + assert profile["correction_factors"][0] == pytest.approx(2.0, rel=0.15) + assert profile["correction_factors"][1] == pytest.approx(1.066, rel=0.08) + assert profile["correction_factors"][2] == pytest.approx(1.0, abs=0.02) + + +def test_detect_aperture_correction_star_candidates_finds_numpy_local_peaks(): + sigma = 1.8 + fwhm = GAUSSIAN_SIGMA_TO_FWHM * sigma + y, x = np.mgrid[0:140, 0:140] + image = np.full((140, 140), 10.0, dtype=float) + positions = np.array([ + [30.0, 35.0], + [95.0, 42.0], + [58.0, 108.0], + ]) + amplitudes = np.array([1000.0, 850.0, 700.0]) + for (xc, yc), amplitude in zip(positions, amplitudes): + image += amplitude * np.exp(-((x - xc) ** 2 + (y - yc) ** 2) / (2.0 * sigma ** 2)) + + candidates = detect_aperture_correction_star_candidates(image, fwhm_hint=fwhm) + + assert candidates.shape[0] >= positions.shape[0] + for xc, yc in positions: + nearest = np.min(np.hypot(candidates[:, 0] - xc, candidates[:, 1] - yc)) + assert nearest < 1.5 + + +def test_compute_star_aperture_grid_applies_aperture_correction_factors(): + y, x = np.mgrid[0:41, 0:41] + image = 100.0 * np.exp(-((x - 20.0) ** 2 + (y - 20.0) ** 2) / (2.0 * 2.0 ** 2)) + apertures = np.array([2.5, 4.0]) + annuli = np.array([0.0]) + + raw_flux, _ = compute_star_aperture_grid( + image, + 0, + 20.0, + 20.0, + apertures, + annuli, + ) + corrected_flux, _ = compute_star_aperture_grid( + image, + 0, + 20.0, + 20.0, + apertures, + annuli, + aperture_correction_factors=np.array([2.0, 1.25]), + ) + + np.testing.assert_allclose(corrected_flux[:, 0], raw_flux[:, 0] * np.array([2.0, 1.25])) + + +def test_compute_star_aperture_grid_ignores_invalid_aperture_geometry(): + image = np.ones((20, 20), dtype=float) + + flux, bg = compute_star_aperture_grid( + image, + 0, + 10.0, + 10.0, + np.array([np.nan]), + np.array([0.0]), + ) + + assert flux.shape == (1, 1) + assert bg.shape == (1, 1) + assert np.isnan(flux[0, 0]) + assert np.isnan(bg[0, 0]) + + +def test_aperture_contains_overexposed_pixel_checks_aperture_only(monkeypatch): + import exotic.exotic as exotic_module + + class FakeMask: + def __init__(self, xc, yc, radius): + self.x0 = int(np.floor(xc - radius)) + self.x1 = int(np.ceil(xc + radius)) + 1 + self.y0 = int(np.floor(yc - radius)) + self.y1 = int(np.ceil(yc + radius)) + 1 + y, x = np.mgrid[self.y0:self.y1, self.x0:self.x1] + self.data = (((x - xc) ** 2 + (y - yc) ** 2) <= radius ** 2).astype(float) + + def cutout(self, data): + return np.asarray(data)[self.y0:self.y1, self.x0:self.x1] + + class FakeCircularAperture: + def __init__(self, positions, r): + self.xc, self.yc = positions[0] + self.r = r + + def to_mask(self, method="exact"): + return [FakeMask(self.xc, self.yc, self.r)] + + monkeypatch.setattr(exotic_module, "CircularAperture", FakeCircularAperture) + + data = np.zeros((20, 20), dtype=float) + data[10, 10] = 90.0 + data[2, 2] = 100.0 + + assert aperture_contains_overexposed_pixel(data, 10.0, 10.0, 2.5, 80.0) is True + assert aperture_contains_overexposed_pixel(data, 10.0, 10.0, 2.5, 95.0) is False + assert aperture_contains_overexposed_pixel(data, 10.0, 10.0, 2.5, 90.0) is False + + +def test_populate_aperture_data_skips_field_star_corrections_when_disabled(monkeypatch): + import exotic.exotic as exotic_module + + def fail_field_star_estimate(*_args, **_kwargs): + raise AssertionError("field-star FWHM estimation should be opt-in") + + monkeypatch.setattr(exotic_module, "estimate_isolated_field_star_psfs", fail_field_star_estimate) + y, x = np.mgrid[0:41, 0:41] + image = 100.0 * np.exp(-((x - 20.0) ** 2 + (y - 20.0) ** 2) / (2.0 * 2.0 ** 2)) + psf_data = { + "target": np.array([[20.0, 20.0, 100.0, 2.0, 2.0, 0.0, 0.0]]), + } + aper_data = initialize_aperture_data_store( + frame_count=1, + aperture_count=1, + annulus_count=1, + comp_star_count=0, + ) + + profile = populate_aperture_data_for_frame( + image, + 0, + psf_data, + 0, + aper_data, + np.array([4.0]), + np.array([0.0]), + fast_aperture_mask=False, + use_aperture_corrections_and_full_image_fwhm=False, + ) + + assert profile["applied"] is False + assert np.isfinite(aper_data["target"][0, 0, 0]) + + +def test_stellar_variability_aperture_estimation_uses_first_five_vetted_comparisons(): + science_comp_stars = [[float(index), float(index + 100)] for index in range(8)] + + assert aperture_estimation_comparison_stars( + science_comp_stars, + stellar_variability_only=False, + ) == science_comp_stars + assert aperture_estimation_comparison_stars( + science_comp_stars, + stellar_variability_only=True, + ) == science_comp_stars[:5] + + +def test_stellar_variability_aperture_grid_excludes_variable_target_and_uses_comp_seeing(monkeypatch): + import exotic.exotic as exotic_module + + measured = [] + + def fake_compute_star_aperture_grid( + _data, + star_index, + _xc, + _yc, + apertures, + annuli, + **_kwargs, + ): + aperture_values = np.asarray(apertures, dtype=float).reshape(-1) + annulus_values = np.asarray(annuli, dtype=float).reshape(-1) + measured.append((star_index, aperture_values.copy(), annulus_values.copy())) + shape = (len(aperture_values), len(annulus_values)) + flux = np.full(shape, 100.0 + star_index, dtype=float) + background = np.full(shape, 10.0 + star_index, dtype=float) + noise = { + component: np.ones(shape, dtype=float) + for component in exotic_module.NOISE_BUDGET_COMPONENT_KEYS + } + return flux, background, noise + + monkeypatch.setattr(exotic_module, "compute_star_aperture_grid", fake_compute_star_aperture_grid) + psf_data = { + # The VSX science target deliberately has very different seeing. It must not + # determine the stellar-variability aperture grid. + "target": np.array([[10.0, 10.0, 100.0, 9.0, 9.0, 0.0, 0.0]]), + "comp1": np.array([[12.0, 10.0, 90.0, 2.0, 2.0, 0.0, 0.0]]), + "comp2": np.array([[14.0, 10.0, 80.0, 4.0, 4.0, 0.0, 0.0]]), + } + assert aperture_frame_sigma_from_psf_data( + psf_data, + 0, + comparison_indices=[0, 1], + ) == pytest.approx(3.0) + + full_grid = initialize_aperture_data_store(1, 1, 1, 2) + populate_aperture_data_for_frame( + np.zeros((25, 25), dtype=float), + 0, + psf_data, + 2, + full_grid, + np.array([2.0]), + np.array([8.0]), + fast_aperture_mask=False, + adaptive_apertures=True, + comp_indices=[0, 1], + include_target=False, + frame_sigma_comp_indices=[0, 1], + ) + + assert [star_index for star_index, _apers, _annuli in measured] == [1, 2] + assert all(apers[0] == pytest.approx(6.0) for _index, apers, _annuli in measured) + assert np.all(np.isnan(full_grid["target"])) + + frozen = collapse_aperture_data_to_selected_grid_cell(full_grid, 0, 0) + measured.clear() + populate_aperture_data_for_frame( + np.zeros((25, 25), dtype=float), + 0, + psf_data, + 2, + frozen, + np.array([2.0]), + np.array([8.0]), + fast_aperture_mask=False, + adaptive_apertures=True, + comp_indices=[], + include_target=True, + frame_sigma_comp_indices=[0, 1], + ) + + assert len(measured) == 1 + assert measured[0][0] == 0 + assert measured[0][1][0] == pytest.approx(6.0) + assert frozen["target"][0, 0, 0] == pytest.approx(100.0) + + +def test_frozen_aperture_path_only_grids_estimators_then_backfills_additional_stars(monkeypatch): + import exotic.exotic as exotic_module + + measured_star_indices = [] + + def fake_compute_star_aperture_grid( + _data, + star_index, + _xc, + _yc, + apertures, + annuli, + **_kwargs, + ): + measured_star_indices.append(star_index) + shape = (len(np.asarray(apertures).reshape(-1)), len(np.asarray(annuli).reshape(-1))) + flux = np.full(shape, 100.0 + star_index, dtype=float) + background = np.full(shape, 10.0 + star_index, dtype=float) + noise = { + component: np.full(shape, 1.0 + star_index, dtype=float) + for component in exotic_module.NOISE_BUDGET_COMPONENT_KEYS + } + return flux, background, noise + + monkeypatch.setattr(exotic_module, "compute_star_aperture_grid", fake_compute_star_aperture_grid) + psf_data = { + "target": np.array([[10.0, 10.0, 100.0, 2.0, 2.0, 0.0, 0.0]]), + "comp1": np.array([[12.0, 10.0, 90.0, 2.0, 2.0, 0.0, 0.0]]), + "comp2": np.array([[14.0, 10.0, 80.0, 2.0, 2.0, 0.0, 0.0]]), + "comp3": np.array([[16.0, 10.0, 70.0, 2.0, 2.0, 0.0, 0.0]]), + "comp4": np.array([[18.0, 10.0, 60.0, 2.0, 2.0, 0.0, 0.0]]), + } + full_grid = initialize_aperture_data_store(1, 2, 2, 4) + populate_aperture_data_for_frame( + np.zeros((25, 25), dtype=float), + 0, + psf_data, + 4, + full_grid, + np.array([3.0, 4.0]), + np.array([8.0, 10.0]), + fast_aperture_mask=False, + comp_indices=[0, 1], + ) + + assert measured_star_indices == [0, 1, 2] + assert np.all(np.isfinite(full_grid["target"])) + assert np.all(np.isfinite(full_grid["comp1"])) + assert np.all(np.isfinite(full_grid["comp2"])) + assert np.all(np.isnan(full_grid["comp3"])) + assert np.all(np.isnan(full_grid["comp4"])) + + frozen = collapse_aperture_data_to_selected_grid_cell(full_grid, 1, 0) + measured_star_indices.clear() + populate_aperture_data_for_frame( + np.zeros((25, 25), dtype=float), + 0, + psf_data, + 4, + frozen, + np.array([4.0]), + np.array([8.0]), + fast_aperture_mask=False, + comp_indices=[2, 3], + include_target=False, + ) + + assert measured_star_indices == [3, 4] + assert frozen["target"].shape == (1, 1, 1) + assert frozen["target"][0, 0, 0] == pytest.approx(100.0) + assert frozen["comp1"][0, 0, 0] == pytest.approx(101.0) + assert frozen["comp2"][0, 0, 0] == pytest.approx(102.0) + assert frozen["comp3"][0, 0, 0] == pytest.approx(103.0) + assert frozen["comp4"][0, 0, 0] == pytest.approx(104.0) + + +def test_should_use_fast_target_centroid_disables_fast_sigma_path_for_adaptive_runs(): + assert should_use_fast_target_centroid(1, adaptive_apertures=False) is True + assert should_use_fast_target_centroid(6, adaptive_apertures=False) is False + assert should_use_fast_target_centroid(1, adaptive_apertures=True) is False + + +def test_resolve_frame_aperture_radii_scales_sigma_grid(): + apertures, annuli = resolve_frame_aperture_radii( + np.array([2.0, 3.0]), + np.array([8.0, 10.0]), + adaptive_apertures=True, + frame_sigma=1.5, + fallback_sigma=1.0, + ) + + assert np.allclose(apertures, np.array([3.0, 4.5])) + assert np.allclose(annuli, np.array([12.0, 15.0])) + + +def test_resolve_frame_aperture_radii_accepts_scalar_values(): + apertures, annuli = resolve_frame_aperture_radii( + 2.0, + 8.0, + adaptive_apertures=True, + frame_sigma=1.5, + fallback_sigma=1.0, + ) + + assert apertures.shape == (1,) + assert annuli.shape == (1,) + assert apertures[0] == pytest.approx(3.0) + assert annuli[0] == pytest.approx(12.0) + + +def test_resolve_sky_annulus_geometry_enforces_fwhm_floor_and_min_sky_pixels(): + geometry = resolve_sky_annulus_geometry(aperture_radius=1.5, annulus_width=2.0, psf_sigma=1.0) + + assert geometry["inner_radius"] == pytest.approx(3.0 * 2.355) + assert geometry["effective_sky_pixels"] == pytest.approx(250.0, abs=1e-9) + assert geometry["annulus_width"] > 2.0 + + +def test_choose_centroid_seed_position_prefers_previous_fit_for_small_predicted_jumps(): + seed = choose_centroid_seed_position([100.8, 200.2], previous_psf_row=np.array([100.2, 199.9, 1, 1, 1, 0, 0])) + np.testing.assert_allclose(seed, np.array([100.2, 199.9])) + + +def test_choose_centroid_seed_position_falls_back_to_predicted_for_large_jump_or_invalid_previous(): + seed_far = choose_centroid_seed_position([110.0, 210.0], previous_psf_row=np.array([100.0, 200.0, 1, 1, 1, 0, 0])) + seed_nan = choose_centroid_seed_position([110.0, 210.0], previous_psf_row=np.array([np.nan, 200.0, 1, 1, 1, 0, 0])) + + np.testing.assert_allclose(seed_far, np.array([110.0, 210.0])) + np.testing.assert_allclose(seed_nan, np.array([110.0, 210.0])) + + +def test_representative_psf_sigma_uses_valid_frames_and_fallback(): + psf_rows = np.array([ + [0.0, 0.0, 1.0, 2.0, 2.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 2.2, 1.8, 0.0, 0.0], + [0.0, 0.0, 1.0, np.nan, np.nan, 0.0, 0.0], + ]) + + assert np.isclose(representative_psf_sigma(psf_rows, fallback_sigma=1.0), 2.0) + assert np.isclose(representative_psf_sigma(np.full((0, 7), np.nan), fallback_sigma=1.25), 1.25) + + +def test_summarize_adaptive_aperture_usage_reports_frame_scaled_stats(): + psf_rows = np.array([ + [0.0, 0.0, 1.0, 2.0, 2.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 3.0, 3.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 4.0, 4.0, 0.0, 0.0], + ]) + + summary = summarize_adaptive_aperture_usage(psf_rows, aperture_scale=2.5, annulus_scale=9.0, fallback_sigma=1.0) + + np.testing.assert_allclose(summary["aperture_series"], np.array([5.0, 7.5, 10.0])) + np.testing.assert_allclose(summary["annulus_series"], np.array([18.0, 27.0, 36.0])) + np.testing.assert_allclose(summary["fwhm_series"], np.array([4.71, 7.065, 9.42])) + assert np.isclose(summary["aperture_median"], 7.5) + assert np.isclose(summary["aperture_std"], np.std([5.0, 7.5, 10.0])) + assert np.isclose(summary["aperture_min"], 5.0) + assert np.isclose(summary["aperture_max"], 10.0) + assert summary["aperture_sigma"] == 2.5 + assert summary["annulus_sigma"] == 9.0 + + +def test_adaptive_aperture_outlier_mask_rejects_isolated_spike_but_keeps_repeated_lower_mode(): + aperture_series = np.array([10.0, 10.1, 9.4, 10.0, 9.4, 10.1, 10.0, 15.2, 10.1, 9.4, 10.0, 10.1]) + annulus_series = aperture_series * 3.0 + + mask = adaptive_aperture_outlier_mask(aperture_series, annulus_series) + + expected = np.zeros_like(aperture_series, dtype=bool) + expected[7] = True + np.testing.assert_array_equal(mask, expected) + + +def test_auto_tune_aperture_grid_uses_comparison_field_consistency(): + coarse_apertures_sigma = np.array([2.0, 3.0]) + coarse_annuli_sigma = np.array([8.0]) + coarse_aper_data = { + "target": np.array([[[10.0]], [[11.0]], [[12.0]], [[13.0]], [[14.0]], [[15.0]]]), + "comp1": np.array([[[5.0], [5.0]], [[5.0], [5.0]], [[5.0], [5.0]], [[5.0], [5.0]], [[5.0], [5.0]], [[5.0], [5.0]]]), + "comp2": np.array([[[7.5], [7.5]], [[7.5], [7.5]], [[7.5], [12.0]], [[7.5], [7.5]], [[7.5], [7.5]], [[7.5], [7.5]]]), + } + subset_airmass = np.arange(1.0, 7.0) + + _, _, best_candidate, _ = auto_tune_aperture_sigma_grid( + coarse_apertures_sigma, + coarse_annuli_sigma, + coarse_aper_data, + comp_star_count=2, + subset_airmass=subset_airmass, + require_comp_star=True, + ) + + assert best_candidate["aper_sigma"] == 2.0 + assert best_candidate["comp_index"] in (0, 1) + + +def test_fit_lightcurve_to_every_comparison_candidate_uses_selected_aperture(monkeypatch): + calls = [] + + class DummyFit: + def __init__(self, size): + self.residuals = np.full(size, 0.01) + self.data = np.ones(size) + + def fake_fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times=None, **kwargs): + calls.append({ + "times": np.asarray(times), + "tflux": np.asarray(tflux), + "cflux": np.asarray(cflux), + "jd_times": np.asarray(jd_times), + "kwargs": dict(kwargs), + }) + return DummyFit(len(times)), np.asarray(tflux), np.asarray(cflux) + + monkeypatch.setattr("exotic.exotic.fit_lightcurve", fake_fit_lightcurve) + + times = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]) + jd_times = np.array([11.0, 12.0, 13.0, 14.0, 15.0, 16.0]) + airmass = np.array([1.1, 1.2, 1.3, 1.4, 1.5, 1.6]) + aper_data = { + "target": np.array([ + [[1.0], [10.0]], + [[2.0], [11.0]], + [[3.0], [12.0]], + [[4.0], [13.0]], + [[5.0], [14.0]], + [[6.0], [15.0]], + ]), + "comp1": np.array([ + [[4.0], [20.0]], + [[5.0], [np.nan]], + [[6.0], [22.0]], + [[7.0], [23.0]], + [[8.0], [24.0]], + [[9.0], [25.0]], + ]), + "comp2": np.array([ + [[7.0], [30.0]], + [[8.0], [31.0]], + [[9.0], [32.0]], + [[10.0], [33.0]], + [[11.0], [34.0]], + [[12.0], [35.0]], + ]), + } + photometry_info = { + "best_fit_lc": object(), + "comp_star_num": 2, + "min_aperture": 5.0, + "min_annulus": 12.0, + "aperture_index": 1, + "annulus_index": 0, + } + + candidate_fits = fit_lightcurve_to_every_comparison_candidate( + times, + jd_times, + airmass, + ld=np.array([0.1, 0.2, 0.3, 0.4]), + p_dict={"rprs": 0.1}, + comp_stars=[[100, 200], [300, 400]], + psf_data={}, + aper_data=aper_data, + photometry_info=photometry_info, + ) + + assert len(candidate_fits) == 2 + assert candidate_fits[0]["selected"] is False + assert candidate_fits[1]["selected"] is True + assert calls[0]["kwargs"]["final_fit_mode"] == "ns" + assert calls[1]["kwargs"]["final_fit_mode"] == "ns" + np.testing.assert_array_equal(calls[0]["times"], np.array([1.0, 3.0, 4.0, 5.0, 6.0])) + np.testing.assert_array_equal(calls[0]["tflux"], np.array([10.0, 12.0, 13.0, 14.0, 15.0])) + np.testing.assert_array_equal(calls[0]["cflux"], np.array([20.0, 22.0, 23.0, 24.0, 25.0])) + np.testing.assert_array_equal(calls[0]["jd_times"], np.array([11.0, 13.0, 14.0, 15.0, 16.0])) + np.testing.assert_array_equal(calls[1]["times"], np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0])) + np.testing.assert_array_equal(calls[1]["tflux"], np.array([10.0, 11.0, 12.0, 13.0, 14.0, 15.0])) + np.testing.assert_array_equal(calls[1]["cflux"], np.array([30.0, 31.0, 32.0, 33.0, 34.0, 35.0])) + + +def test_fit_lightcurve_to_every_comparison_candidate_records_sparse_candidate_failure(monkeypatch): + calls = [] + + class DummyFit: + def __init__(self, size): + self.residuals = np.full(size, 0.01) + self.data = np.ones(size) + + def fake_fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times=None, **kwargs): + calls.append({ + "times": np.asarray(times), + "tflux": np.asarray(tflux), + "cflux": np.asarray(cflux), + "jd_times": np.asarray(jd_times), + "kwargs": dict(kwargs), + }) + return DummyFit(len(times)), np.asarray(tflux), np.asarray(cflux) + + monkeypatch.setattr("exotic.exotic.fit_lightcurve", fake_fit_lightcurve) + + times = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]) + jd_times = np.array([11.0, 12.0, 13.0, 14.0, 15.0, 16.0]) + airmass = np.array([1.1, 1.2, 1.3, 1.4, 1.5, 1.6]) + aper_data = { + "target": np.array([ + [[10.0]], + [[11.0]], + [[12.0]], + [[13.0]], + [[14.0]], + [[15.0]], + ]), + "comp1": np.array([ + [[20.0]], + [[np.nan]], + [[np.nan]], + [[np.nan]], + [[np.nan]], + [[np.nan]], + ]), + "comp2": np.array([ + [[30.0]], + [[31.0]], + [[32.0]], + [[33.0]], + [[34.0]], + [[35.0]], + ]), + } + photometry_info = { + "best_fit_lc": object(), + "comp_star_num": 2, + "min_aperture": 5.0, + "min_annulus": 12.0, + "aperture_index": 0, + "annulus_index": 0, + } + + candidate_fits = fit_lightcurve_to_every_comparison_candidate( + times, + jd_times, + airmass, + ld=np.array([0.1, 0.2, 0.3, 0.4]), + p_dict={"rprs": 0.1}, + comp_stars=[[100, 200], [300, 400]], + psf_data={}, + aper_data=aper_data, + photometry_info=photometry_info, + ) + + assert len(calls) == 1 + assert candidate_fits[0]["fit"] is None + assert candidate_fits[0]["coverage_rejected"] is True + assert candidate_fits[0]["fit_diagnostics"]["failed_stage"] == "coverage" + assert "low-coverage clipping" in candidate_fits[0]["failure_reason"] + assert candidate_fits[1]["fit"] is not None + assert candidate_fits[1]["coverage_rejected"] is False + assert candidate_fits[1]["failure_reason"] is None + assert calls[0]["kwargs"]["final_fit_mode"] == "ns" + + +def test_log_comparison_candidate_fit_summaries_includes_reasons(monkeypatch): + logged = [] + monkeypatch.setattr("exotic.exotic.log_info", lambda message, warn=False, error=False: logged.append(message)) + + candidate_fit_summaries = [ + { + "label": "Comp 1", + "position": [100, 200], + "selected": False, + "fit": None, + "res_std": np.inf, + "eebls_snr": np.nan, + "coverage_count": 1, + "coverage_total_frame_count": 3, + "coverage_reference_count": 3.0, + "coverage_min_required_count": 2, + "fit_point_count": 0, + "fit_diagnostics": {"usable_point_count": 0}, + "failure_reason": "comparison candidate rejected after iterative low-coverage clipping (1 < 2 valid frame(s); peer median=3.0).", + }, + { + "label": "Comp 2", + "position": [300, 400], + "selected": True, + "fit": object(), + "eebls_snr": 6.5, + "transit_delta_bic": 18.4, + "residual_scatter": 0.0042, + "ktmf_metric": 4.35, + "ktmf_contributions": [ + { + "label": "EEBLS Depth SNR", + "available": True, + "points": 1.25, + "max_points": 1.40, + "score": 0.89, + "detail": "6.50", + } + ], + "coverage_count": 3, + "coverage_total_frame_count": 3, + "coverage_reference_count": 3.0, + "coverage_min_required_count": 2, + "fit_point_count": 3, + "fit_diagnostics": {"usable_point_count": 3}, + "failure_reason": None, + "parameter_summary": "fit_method=ultranest, Tmid=1.0 +/- 0.1", + }, + ] + + log_comparison_candidate_fit_summaries( + candidate_fit_summaries, + { + "selection_basis": "comparison_field", + "selection_metric": "ktmf", + "comp_star_num": 2, + "comparison_ktmf_metric": 4.35, + "comparison_eebls_snr": 6.5, + "comparison_transit_delta_bic": 18.4, + }, + ) + + assert any("Selection basis: comparison-field" in message for message in logged) + assert any("Selection metric: KTMF" in message for message in logged) + assert any("coverage=1 valid frame(s) out of 3 total; min_required=2; peer_median=3.0" in message for message in logged) + assert any("Comp 1" in message and "reason=comparison candidate rejected after iterative low-coverage clipping" in message for message in logged) + assert any("Comp 2 [selected]" in message and "ktmf=4.35/5.00" in message and "comparison-field calibration ranked this star best" in message for message in logged) + assert any("KTMF contribution: EEBLS Depth SNR +1.25/1.40" in message for message in logged) + assert any("parameters: fit_method=ultranest" in message for message in logged) + + +def test_log_comparison_calibration_fit_attempt_summaries_includes_reasons(monkeypatch): + logged = [] + monkeypatch.setattr("exotic.exotic.log_info", lambda message, warn=False, error=False: logged.append(message)) + + attempts = [ + { + "label": "Comp 1", + "position": [100, 200], + "selected": False, + "aggregate_score": 0.01, + "coverage_count": 3, + "coverage_total_frame_count": 3, + "coverage_reference_count": 3.0, + "coverage_min_required_count": 2, + "fit": None, + "res_std": np.inf, + "eebls_snr": np.nan, + "fit_point_count": 0, + "fit_diagnostics": {"usable_point_count": 0}, + "failure_reason": "relative-flux filtering left 0 usable point(s); rejected 3/3 frame(s) during invalid target/reference ratio screening (non-finite=0, non-positive=3, finite ratio range=-1.0000 to -1.0000).", + "parameter_summary": None, + }, + { + "label": "Comp 2", + "position": [300, 400], + "selected": True, + "aggregate_score": 0.02, + "coverage_count": 3, + "coverage_total_frame_count": 3, + "coverage_reference_count": 3.0, + "coverage_min_required_count": 2, + "fit": object(), + "eebls_snr": 5.2, + "transit_delta_bic": 18.4, + "residual_scatter": 0.0035, + "ktmf_metric": 4.60, + "ktmf_contributions": [ + { + "label": "Residual Scatter Around Full Model Fit", + "available": True, + "points": 0.63, + "max_points": 0.80, + "score": 0.79, + "detail": "0.3500%", + } + ], + "fit_point_count": 3, + "fit_diagnostics": {"usable_point_count": 3}, + "failure_reason": None, + "parameter_summary": "fit_method=ultranest, Tmid=1.0 +/- 0.1", + }, + ] + + log_comparison_calibration_fit_attempt_summaries(attempts, "Aperture photometry (aper=7.05px, annulus=22.73px)") + + assert any("Comparison-star calibration target-fit diagnostics:" in message for message in logged) + assert any("Photometry method: Aperture photometry (aper=7.05px, annulus=22.73px)" in message for message in logged) + assert any("Comp 1" in message and "reason=relative-flux filtering left 0 usable point(s)" in message for message in logged) + assert any("Comp 2 [selected]" in message and "ktmf=4.60/5.00" in message and "fit_points=3" in message for message in logged) + assert any("KTMF contribution: Residual Scatter Around Full Model Fit +0.63/0.80" in message for message in logged) + assert any("parameters: fit_method=ultranest" in message for message in logged) + + +def test_log_target_fit_candidate_summaries_includes_methods_and_reasons(monkeypatch): + logged = [] + monkeypatch.setattr("exotic.exotic.log_info", lambda message, warn=False, error=False: logged.append(message)) + + candidate_summaries = [ + { + "label": "Comp 1", + "position": [100, 200], + "selected": False, + "method_label": "Aperture photometry (aper=7.05px, annulus=22.73px)", + "prescore": 0.005, + "fit": None, + "residual_scatter": np.inf, + "eebls_snr": np.nan, + "ktmf_metric": 0.0, + "ktmf_contributions": [ + { + "label": "Deviation From Expected Value", + "available": False, + "points": 0.0, + "max_points": 0.0, + "score": np.nan, + "detail": "expected-value deviation disabled or unavailable", + } + ], + "coverage_count": 3, + "coverage_total_frame_count": 3, + "coverage_reference_count": 3.0, + "coverage_min_required_count": 2, + "fit_point_count": 0, + "fit_diagnostics": {"usable_point_count": 0}, + "failure_reason": "relative-flux filtering left 0 usable point(s); rejected 3/3 frame(s) during invalid target/reference ratio screening (non-finite=0, non-positive=3, finite ratio range=-1.0000 to -1.0000).", + "parameter_summary": None, + }, + ] + + log_target_fit_candidate_summaries(candidate_summaries) + + assert any("Target-fit candidate diagnostics:" in message for message in logged) + assert any( + "Comp 1" in message + and "with Aperture photometry (aper=7.05px, annulus=22.73px)" in message + and "reason=relative-flux filtering left 0 usable point(s)" in message + for message in logged + ) + assert any("KTMF contribution: Deviation From Expected Value +0.00/0.00 (unavailable;" in message for message in logged) + + +def test_compute_transit_qc_ktmf_uses_rebalanced_component_weights(): + summary = { + "delta_bic": 10.0, + "delta_chi2": 50.0, + "deviation_from_expected_value": 0.6, + "tmid_deviation_sigma": 1.0, + "rprs_deviation_sigma": 2.0, + "residual_scatter": 0.005, + "transit_depth_for_residual_scatter": 0.01, + "residual_scatter_to_depth_ratio": 0.5, + "residual_flatness_score": 0.5, + "residual_flatness_detail": "curve=0.50", + "tmid_gaussianity_score": 0.8, + "tmid_gaussianity_score_uncertainty": 0.04, + "tmid_gaussianity_detail": "strongly Gaussian-like", + "rprs_sigma": 6.0, + "duration_ratio": 1.0, + "eebls_depth_snr": 8.0, + } + + ktmf_metric, contributions = compute_transit_qc_ktmf(summary) + contributions_by_label = {contribution["label"]: contribution for contribution in contributions} + + assert "Model Evidence" not in contributions_by_label + assert "Delta BIC" not in contributions_by_label + assert "Delta chi2" not in contributions_by_label + scale = 5.0 / (2.0 + 0.7 + 1.0 + 1.0 + 0.75 + 1.3) + assert contributions_by_label["Deviation From Expected Value"]["max_points"] == pytest.approx(2.0 * scale) + assert contributions_by_label["Residual Scatter Around Full Model Fit"]["max_points"] == pytest.approx(0.7 * scale) + assert contributions_by_label["Residual Flatness"]["max_points"] == pytest.approx(1.0 * scale) + assert contributions_by_label["Tmid Posterior Gaussianity"]["max_points"] == pytest.approx(1.0 * scale) + assert contributions_by_label["Tmid Posterior Gaussianity"]["score_uncertainty"] == pytest.approx(0.04) + assert "Rp/R* Significance" not in contributions_by_label + assert contributions_by_label["Duration Consistency"]["max_points"] == pytest.approx(0.75 * scale) + assert contributions_by_label["EEBLS Depth SNR"]["max_points"] == pytest.approx(1.3 * scale) + assert "Rp/R* sigma=2.00" in contributions_by_label["Deviation From Expected Value"]["detail"] + assert "Tmid" not in contributions_by_label["Deviation From Expected Value"]["detail"] + assert contributions_by_label["Residual Flatness"]["score"] == pytest.approx(0.5) + + expected_ktmf = scale * ( + 2.0 * 0.6 + + 0.7 * 1.0 + + 1.0 * 0.5 + + 1.0 * 0.8 + + 0.75 * 1.0 + + 1.3 * (1.0 - np.exp(-2.0)) + ) + assert ktmf_metric == pytest.approx(expected_ktmf) + + +class _TmidPosteriorFit: + def __init__(self, values, weights=None, sampled_keys=("tmid",)): + self.values = np.asarray(values, dtype=float) + self.weights = None if weights is None else np.asarray(weights, dtype=float) + self.sampled_keys = list(sampled_keys) + self.bounds = {"tmid": [float(np.min(self.values)), float(np.max(self.values))]} + + def _get_triangle_plot_samples(self): + return self.values[:, np.newaxis], np.zeros(self.values.size), self.weights + + +def test_tmid_posterior_gaussianity_distinguishes_gaussian_flat_skewed_and_multimodal_shapes(): + rng = np.random.default_rng(20260715) + center = 2460835.82621 + gaussian_values = center + rng.normal(0.0, 0.0015, 5000) + flat_values = np.linspace(center - 0.006, center + 0.006, 5000) + skewed_values = center + 0.002 * (rng.lognormal(-1.0, 0.5, 5000) - 0.42) + multimodal_values = center + np.concatenate([ + rng.normal(-0.003, 0.0005, 2500), + rng.normal(0.003, 0.0005, 2500), + ]) + + gaussian = transit_qc_tmid_gaussianity_summary(_TmidPosteriorFit(gaussian_values)) + flat = transit_qc_tmid_gaussianity_summary(_TmidPosteriorFit(flat_values)) + skewed = transit_qc_tmid_gaussianity_summary(_TmidPosteriorFit(skewed_values)) + multimodal = transit_qc_tmid_gaussianity_summary(_TmidPosteriorFit(multimodal_values)) + + assert gaussian["available"] is True + assert gaussian["score"] > 0.90 + assert np.isfinite(gaussian["score_uncertainty"]) + assert "strongly Gaussian-like" in gaussian["detail"] + assert flat["score"] < 0.05 + assert skewed["score"] < 0.60 + assert multimodal["score"] < 0.20 + + +def test_tmid_posterior_gaussianity_uses_ultranest_sample_weights(): + rng = np.random.default_rng(717) + center = 2460835.82621 + flat_values = np.linspace(center - 0.01, center + 0.01, 6000) + gaussian_values = center + rng.normal(0.0, 0.001, 2500) + values = np.concatenate([flat_values, gaussian_values]) + weights = np.concatenate([ + np.full(flat_values.size, 1e-8), + np.ones(gaussian_values.size), + ]) + + summary = transit_qc_tmid_gaussianity_summary(_TmidPosteriorFit(values, weights=weights)) + + assert summary["available"] is True + assert summary["effective_sample_count"] == pytest.approx(2500.0, rel=1e-4) + assert summary["score"] > 0.85 + + +def test_tmid_posterior_gaussianity_is_unavailable_when_tmid_was_fixed(): + summary = transit_qc_tmid_gaussianity_summary( + _TmidPosteriorFit(np.linspace(0.0, 1.0, 500), sampled_keys=()) + ) + + assert summary["available"] is False + assert not np.isfinite(summary["score"]) + assert "fixed rather than sampled" in summary["detail"] + + +def test_transit_qc_residual_scatter_score_full_credit_floor_and_zero_ceiling(): + transit_depth = 0.02 + assert transit_qc_residual_scatter_score(0.0, transit_depth) == pytest.approx(1.0) + assert transit_qc_residual_scatter_score(0.01, transit_depth) == pytest.approx(1.0) + assert transit_qc_residual_scatter_score(0.08, transit_depth) == pytest.approx(0.0) + assert transit_qc_residual_scatter_score(0.09, transit_depth) == pytest.approx(0.0) + assert not np.isfinite(transit_qc_residual_scatter_score(0.005)) + + mid_score = transit_qc_residual_scatter_score(0.03, transit_depth) + assert 0.0 < mid_score < 1.0 + assert mid_score < transit_qc_residual_scatter_score(0.02, transit_depth) + + +def test_evaluate_transit_detection_qc_uses_transit_component_depth_for_residual_scatter(): + transit_component = np.ones(21, dtype=float) + transit_component[8:13] = 0.984 + baseline_trend = np.linspace(0.0, 0.04, transit_component.size) + full_model = transit_component + baseline_trend + data = full_model + np.array( + [ + 0.0002, -0.0001, 0.0001, -0.0002, 0.0000, 0.0001, -0.0001, + 0.0002, -0.0002, 0.0001, -0.0001, 0.0002, -0.0002, 0.0001, + 0.0000, -0.0001, 0.0002, -0.0001, 0.0001, 0.0000, -0.0001, + ], + dtype=float, + ) + fit = types.SimpleNamespace( + data=data, + dataerr=np.full(data.shape[0], 0.0015, dtype=float), + model=full_model, + transit=transit_component, + airmass=np.ones(data.shape[0], dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.10, "tmid": 0.5, "inc": 89.0, "a2": 0.0}, + errors={"rprs": 0.01, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 1.0], "tmid": [0.4, 0.6], "inc": [80.0, 90.0]}, + duration_expected=5.0, + duration_measured=5.0, + ) + + summary = evaluate_transit_detection_qc(fit) + + assert summary["computed"] is True + assert summary["transit_depth_for_residual_scatter"] == pytest.approx(0.016, abs=5e-4) + assert summary["residual_scatter_to_depth_ratio"] == pytest.approx( + summary["residual_scatter"] / summary["transit_depth_for_residual_scatter"] + ) + + +def test_transit_qc_residual_flatness_summary_penalizes_residual_structure(): + phase = np.linspace(-0.05, 0.05, 80) + alternating_noise = 0.001 * np.where(np.arange(phase.size) % 2 == 0, -1.0, 1.0) + + flat_summary = transit_qc_residual_flatness_summary(alternating_noise, phase) + trend_summary = transit_qc_residual_flatness_summary( + alternating_noise + 0.004 * np.linspace(-1.0, 1.0, phase.size), + phase, + ) + curve_summary = transit_qc_residual_flatness_summary( + alternating_noise + 0.004 * np.sin(2.0 * np.pi * np.linspace(0.0, 1.0, phase.size)), + phase, + ) + smooth_bowl_summary = transit_qc_residual_flatness_summary( + alternating_noise + 0.003 * np.maximum(0.0, 1.0 - (phase / 0.02) ** 2), + phase, + ) + heteroscedastic_summary = transit_qc_residual_flatness_summary( + alternating_noise * np.r_[np.ones(40), np.full(40, 4.0)], + phase, + ) + one_sided_summary = transit_qc_residual_flatness_summary( + alternating_noise - 0.003, + phase, + ) + + assert flat_summary["available"] is True + assert flat_summary["score"] > 0.9 + assert trend_summary["score"] < flat_summary["score"] + assert trend_summary["score"] < 0.5 + assert curve_summary["score"] < flat_summary["score"] + assert curve_summary["score"] < 0.6 + assert smooth_bowl_summary["score"] < flat_summary["score"] + assert smooth_bowl_summary["score"] < 0.5 + assert smooth_bowl_summary["dominant"] == "curvature/sinusoid" + assert heteroscedastic_summary["score"] < flat_summary["score"] + assert heteroscedastic_summary["score"] < 0.8 + assert one_sided_summary["score"] < flat_summary["score"] + assert one_sided_summary["score"] < 0.4 + assert one_sided_summary["dominant"] == "zero bias" + assert one_sided_summary["sign_imbalance"] > 0.8 + + +def test_transit_qc_residual_flatness_summary_tolerates_one_quiet_patch(): + phase = np.linspace(-0.05, 0.05, 84) + residuals = 0.001 * np.sin(np.arange(phase.size) * 2.3999632) + residuals[-10:] *= 0.12 + + summary = transit_qc_residual_flatness_summary(residuals, phase) + + assert summary["available"] is True + assert summary["score"] > 0.5 + assert summary["scatter_ratio"] < 3.0 + + +def test_transit_qc_sampling_summary_scores_ingress_egress_and_baseline_counts(): + fit = types.SimpleNamespace( + time=np.array([ + -0.090, -0.075, -0.060, + -0.050, -0.045, -0.040, -0.035, -0.030, + -0.020, -0.010, 0.000, 0.010, 0.020, + 0.030, 0.035, 0.040, 0.045, 0.050, + 0.060, 0.075, 0.090, + ]), + parameters={"tmid": 0.0, "rprs": 0.1}, + duration_expected=0.1, + ) + + summary = transit_qc_sampling_summary(fit) + + assert summary["available"] is True + assert summary["ingress_count"] == 5 + assert summary["egress_count"] == 5 + assert summary["in_transit_count"] == 15 + assert summary["pre_baseline_count"] == 3 + assert summary["post_baseline_count"] == 3 + assert 0.0 < summary["score"] < 1.0 + assert "ingress=5, egress=5" in summary["detail"] + + +def test_compute_transit_qc_ktmf_omits_prior_assumed_rprs_component(): + summary = { + "delta_bic": 10.0, + "delta_chi2": 50.0, + "deviation_from_expected_value": 1.0, + "rprs_deviation_sigma": 0.0, + "rprs_prior_assumed": True, + "rprs_prior_assumed_note": "Rp/R* was fixed to the input prior.", + "residual_scatter": 0.005, + "transit_depth_for_residual_scatter": 0.01, + "residual_scatter_to_depth_ratio": 0.5, + "point_count": 86, + "duration_ratio": 1.0, + "eebls_depth_snr": 8.0, + } + + ktmf_metric, contributions = compute_transit_qc_ktmf(summary) + contributions_by_label = {contribution["label"]: contribution for contribution in contributions} + + omitted = contributions_by_label["Deviation From Expected Value"] + assert omitted["available"] is False + assert omitted["points"] == pytest.approx(0.0) + assert omitted["max_points"] == pytest.approx(0.0) + assert "fixed to the input prior" in omitted["detail"] + + assert "Model Evidence" not in contributions_by_label + scale = 5.0 / (0.7 + 0.75 + 1.3) + assert contributions_by_label["Residual Scatter Around Full Model Fit"]["max_points"] == pytest.approx(0.7 * scale) + assert contributions_by_label["Duration Consistency"]["max_points"] == pytest.approx(0.75 * scale) + assert contributions_by_label["EEBLS Depth SNR"]["max_points"] == pytest.approx(1.3 * scale) + + expected_ktmf = scale * ( + 0.7 * 1.0 + + 0.75 * 1.0 + + 1.3 * (1.0 - np.exp(-2.0)) + ) + assert ktmf_metric == pytest.approx(expected_ktmf) + + +def test_compute_transit_qc_ktmf_adds_tmid_gaussianity_for_prior_assumed_geometry(): + summary = { + "geometry_prior_assumed": True, + "geometry_prior_assumed_note": "Transit geometry was fixed to priors.", + "deviation_from_expected_value": 1.0, + "residual_scatter": 0.005, + "transit_depth_for_residual_scatter": 0.01, + "residual_scatter_to_depth_ratio": 0.5, + "point_count": 86, + "duration_ratio": 1.0, + "sampling_score": 1.0, + "sampling_detail": "ingress=4, egress=4", + "eebls_depth_snr": 8.0, + "tmid_gaussianity_score": 0.75, + "tmid_gaussianity_score_uncertainty": 0.05, + "tmid_gaussianity_detail": "broadly Gaussian-like", + } + + ktmf_metric, contributions = compute_transit_qc_ktmf(summary) + contributions_by_label = {contribution["label"]: contribution for contribution in contributions} + + assert contributions_by_label["Deviation From Expected Value"]["available"] is False + assert contributions_by_label["Duration Consistency"]["available"] is False + assert contributions_by_label["Sampling / Cadence"]["available"] is False + assert "fixed to priors" in contributions_by_label["Duration Consistency"]["detail"] + + scale = 5.0 / (0.7 + 1.0 + 1.3) + assert contributions_by_label["Residual Scatter Around Full Model Fit"]["max_points"] == pytest.approx(0.7 * scale) + assert contributions_by_label["Tmid Posterior Gaussianity"]["max_points"] == pytest.approx(1.0 * scale) + assert contributions_by_label["Tmid Posterior Gaussianity"]["score"] == pytest.approx(0.75) + assert contributions_by_label["EEBLS Depth SNR"]["max_points"] == pytest.approx(1.3 * scale) + + expected_ktmf = scale * ( + 0.7 * 1.0 + + 1.0 * 0.75 + + 1.3 * (1.0 - np.exp(-2.0)) + ) + assert ktmf_metric == pytest.approx(expected_ktmf) + + +def test_comparison_candidate_fit_selection_reason_describes_comparison_field_retry(): + reason = comparison_candidate_fit_selection_reason( + { + "selected": True, + "failure_reason": None, + "transit_delta_bic": 18.4, + }, + { + "selection_basis": "comparison_field_retry", + "comp_star_num": 2, + "comparison_transit_delta_bic": 18.4, + }, + ) + + assert "fell back to this star" in reason + + +def test_comparison_calibration_selection_reason_reports_suitability_outlier_rejection(): + reason = comparison_calibration_selection_reason( + { + "selected": False, + "coverage_rejected": False, + "aggregate_score": 0.139668, + "suitability_outlier_rejected": True, + "suitability_high_threshold": 0.0398471675, + }, + best_comp_score=0.021654, + ) + + assert "rejected by high-side sigma clipping" in reason + assert "13.9668%" in reason + + +def test_comparison_star_stability_summary_penalizes_variable_candidates(): + airmass = np.linspace(1.0, 1.5, 6) + summary = comparison_star_stability_summary( + { + "comp1": np.array([100.0, 101.0, 100.5, 101.5, 100.8, 101.2]), + "comp2": np.array([80.0, 80.8, 80.4, 81.0, 80.6, 80.9]), + "comp3": np.array([60.0, 60.4, 84.0, 60.6, 60.5, 60.3]), + }, + airmass, + ) + + assert np.isfinite(summary["field_score"]) + assert summary["best_comp_index"] in (0, 1) + assert summary["comp_summaries"][2]["aggregate_score"] > summary["comp_summaries"][0]["aggregate_score"] + + +def test_exact_comparison_mode_bypasses_star_and_frame_vetting(): + airmass = np.linspace(1.0, 1.5, 8) + summary = comparison_star_stability_summary( + { + "comp1": np.array([100.0, 101.0, 100.0, 101.0, 100.0, 101.0, 100.0, 101.0]), + "comp2": np.array([80.0, 80.0, 80.0, 160.0, 80.0, 80.0, 80.0, 80.0]), + "comp3": np.array([60.0, 60.0, 60.0, 60.0, 60.0, 60.0, np.nan, np.nan]), + }, + airmass, + bypass_vetting=True, + ) + + assert [row["key"] for row in summary["comp_summaries"]] == [ + "comp1", + "comp2", + "comp3", + ] + assert not any(row["coverage_rejected"] for row in summary["comp_summaries"]) + assert not any(row["suitability_outlier_rejected"] for row in summary["comp_summaries"]) + assert np.all(summary["field_image_keep_mask"]) + + +def test_apply_comparison_star_suitability_outlier_rejection_rejects_high_tail(): + comp_summaries = [ + {"label": "Comp 1", "aggregate_score": 0.139668, "coverage_rejected": False}, + {"label": "Comp 2", "aggregate_score": 0.051809, "coverage_rejected": False}, + {"label": "Comp 3", "aggregate_score": 0.024802, "coverage_rejected": False}, + {"label": "Comp 4", "aggregate_score": 0.027680, "coverage_rejected": False}, + {"label": "Comp 5", "aggregate_score": 0.036997, "coverage_rejected": False}, + {"label": "Comp 6", "aggregate_score": 0.037970, "coverage_rejected": False}, + {"label": "Comp 7", "aggregate_score": 0.023458, "coverage_rejected": False}, + {"label": "Comp 8", "aggregate_score": 0.021654, "coverage_rejected": False}, + {"label": "Comp 9", "aggregate_score": 0.025084, "coverage_rejected": False}, + {"label": "Comp 10", "aggregate_score": 0.024918, "coverage_rejected": False}, + ] + + result = apply_comparison_star_suitability_outlier_rejection(comp_summaries) + + assert result["rejected_indices"] == [0, 1] + assert comp_summaries[0]["suitability_outlier_rejected"] is True + assert comp_summaries[1]["suitability_outlier_rejected"] is True + assert comp_summaries[4]["suitability_outlier_rejected"] is False + assert comp_summaries[5]["suitability_outlier_rejected"] is False + assert 0.037970 < result["high_threshold"] < 0.051809 + + +def test_comparison_star_stability_summary_iterates_after_suitability_outlier_rejection(monkeypatch): + monkeypatch.setattr( + "exotic.exotic.normalize_flux_series", + lambda flux_values, validity_mask_func=None: np.asarray(flux_values, dtype=float), + ) + monkeypatch.setattr( + "exotic.exotic.normalized_ratio_series", + lambda flux_a, flux_b: np.array([1.0], dtype=float), + ) + + def fake_build_normalized_comp_ensemble(normalized_flux_map, exclude_key): + comp_index = float(exclude_key.replace("comp", "")) + return np.array([-float(len(normalized_flux_map)), comp_index], dtype=float) + + def fake_prescore(tflux, cflux, airmass, enforce_relative_flux_max=False): + comp_index = int(np.rint(np.asarray(tflux, dtype=float).flat[0])) + reference = np.asarray(cflux, dtype=float).reshape(-1) + if reference.size == 0: + return np.inf + if np.allclose(reference, 1.0): + return 0.0 + if reference[0] < 0: + active_count = int(np.rint(abs(reference[0]))) + ensemble_scores = { + 6: {1: 10.0, 2: 2.5, 3: 1.0, 4: 1.1, 5: 1.2, 6: 1.4}, + 5: {2: 4.0, 3: 1.0, 4: 1.1, 5: 1.2, 6: 1.4}, + 4: {3: 1.0, 4: 1.1, 5: 1.2, 6: 1.4}, + } + return ensemble_scores.get(active_count, {}).get(comp_index, 1.0) + return 0.5 + + monkeypatch.setattr( + "exotic.exotic.build_normalized_comp_ensemble", + fake_build_normalized_comp_ensemble, + ) + monkeypatch.setattr("exotic.exotic.cheap_lightcurve_prescore", fake_prescore) + + summary = comparison_star_stability_summary( + { + "comp1": np.array([1.0], dtype=float), + "comp2": np.array([2.0], dtype=float), + "comp3": np.array([3.0], dtype=float), + "comp4": np.array([4.0], dtype=float), + "comp5": np.array([5.0], dtype=float), + "comp6": np.array([6.0], dtype=float), + }, + np.array([1.0], dtype=float), + ) + + assert summary["suitability_outlier_rejected_count"] == 2 + assert summary["comp_summaries"][0]["suitability_outlier_rejected"] is True + assert summary["comp_summaries"][1]["suitability_outlier_rejected"] is True + assert summary["comp_summaries"][2]["suitability_outlier_rejected"] is False + assert summary["best_comp_index"] == 2 + + +def test_comparison_star_coverage_summary_rejects_sparse_candidates(): + coverage = comparison_star_coverage_summary( + { + "comp1": np.array([100.0, 101.0, 100.5, 101.5, 100.8, 101.2]), + "comp2": np.array([80.0, 80.8, 80.4, 81.0, 80.6, 80.9]), + "comp3": np.array([60.0, np.nan, np.nan, np.nan, np.nan, 60.3]), + } + ) + + assert not coverage["comp1"]["coverage_rejected"] + assert not coverage["comp2"]["coverage_rejected"] + assert coverage["comp3"]["coverage_rejected"] + assert coverage["comp3"]["coverage_count"] == 2 + assert coverage["comp3"]["coverage_total_frame_count"] == 6 + + +def test_comparison_star_coverage_summary_iteratively_rejects_low_count_tail(): + coverage = comparison_star_coverage_summary( + { + "comp1": np.array([10.0] * 10), + "comp2": np.array([11.0] * 10), + "comp3": np.array([12.0] * 10), + "comp4": np.array([13.0] * 7 + [np.nan] * 3), + "comp5": np.array([14.0] * 6 + [np.nan] * 4), + "comp6": np.array([15.0] + [np.nan] * 9), + } + ) + + assert not coverage["comp1"]["coverage_rejected"] + assert not coverage["comp2"]["coverage_rejected"] + assert not coverage["comp3"]["coverage_rejected"] + assert coverage["comp4"]["coverage_rejected"] + assert coverage["comp5"]["coverage_rejected"] + assert coverage["comp6"]["coverage_rejected"] + assert coverage["comp1"]["coverage_total_frame_count"] == 10 + assert coverage["comp1"]["coverage_reference_count"] == pytest.approx(10.0) + assert coverage["comp1"]["coverage_min_required_count"] == 8 + + +def test_comparison_star_coverage_summary_keeps_nearly_complete_candidates(): + frame_count = 146 + coverage = comparison_star_coverage_summary( + { + **{ + f"comp{comp_index + 1}": np.ones(frame_count, dtype=float) + for comp_index in range(8) + }, + "comp9": np.concatenate([np.ones(145, dtype=float), [np.nan]]), + "comp10": np.concatenate([np.ones(142, dtype=float), np.full(4, np.nan)]), + } + ) + + assert coverage["comp9"]["coverage_count"] == 145 + assert coverage["comp10"]["coverage_count"] == 142 + assert coverage["comp9"]["coverage_min_required_count"] == 117 + assert coverage["comp10"]["coverage_min_required_count"] == 117 + assert coverage["comp9"]["coverage_rejected"] is False + assert coverage["comp10"]["coverage_rejected"] is False + + +def test_comparison_star_stability_summary_rejects_low_coverage_candidates(): + airmass = np.linspace(1.0, 1.5, 6) + summary = comparison_star_stability_summary( + { + "comp1": np.array([100.0, 101.0, 100.5, 101.5, 100.8, 101.2]), + "comp2": np.array([80.0, 80.8, 80.4, 81.0, 80.6, 80.9]), + "comp3": np.array([60.0, np.nan, np.nan, np.nan, np.nan, 60.3]), + }, + airmass, + ) + + assert np.isfinite(summary["field_score"]) + assert summary["best_comp_index"] in (0, 1) + assert summary["comp_summaries"][2]["coverage_rejected"] + assert np.isinf(summary["comp_summaries"][2]["aggregate_score"]) + + +def test_comparison_star_stability_summary_rejects_noisy_nearly_complete_candidates_as_outliers(): + frame_count = 146 + airmass = np.linspace(1.0, 1.5, frame_count) + phase = np.linspace(0.0, 4.0 * np.pi, frame_count) + stable_flux = 100.0 * (1.0 + 0.001 * np.sin(phase)) + comp_flux_map = { + f"comp{comp_index + 1}": stable_flux * (1.0 + 0.0001 * comp_index) + for comp_index in range(8) + } + noisy_flux = 100.0 * (1.0 + 0.35 * np.sin(np.linspace(0.0, 14.0 * np.pi, frame_count))) + noisy_flux[-1] = np.nan + choppy_flux = 100.0 * (1.0 + 0.25 * np.sign(np.sin(np.linspace(0.0, 20.0 * np.pi, frame_count)))) + choppy_flux[-4:] = np.nan + comp_flux_map["comp9"] = noisy_flux + comp_flux_map["comp10"] = choppy_flux + + summary = comparison_star_stability_summary(comp_flux_map, airmass) + comp9_summary = summary["comp_summaries"][8] + comp10_summary = summary["comp_summaries"][9] + + assert comp9_summary["coverage_count"] == 145 + assert comp10_summary["coverage_count"] == 142 + assert comp9_summary["coverage_rejected"] is False + assert comp10_summary["coverage_rejected"] is False + assert comp9_summary["suitability_outlier_rejected"] is True + assert comp10_summary["suitability_outlier_rejected"] is True + reason = comparison_calibration_selection_reason( + comp9_summary, + summary["best_comp_score"], + ) + assert "high-side sigma clipping" in reason + assert "low coverage" not in reason + + +def test_comparison_star_stability_summary_rejects_shared_bad_frame(): + airmass = np.linspace(1.0, 1.5, 6) + summary = comparison_star_stability_summary( + { + "comp1": np.array([100.0, 100.8, 99.6, 100.4, 100.1, 140.0]), + "comp2": np.array([80.0, 79.5, 80.6, 80.2, 79.8, 40.0]), + "comp3": np.array([120.0, 121.0, 119.2, 120.5, 119.7, 100.0]), + }, + airmass, + ) + + np.testing.assert_array_equal( + summary["field_image_keep_mask"], + np.array([True, True, True, True, True, False], dtype=bool), + ) + assert summary["image_outlier_rejected_count"] == 1 + assert summary["image_outlier_required_valid_pairs"] == 2 + assert summary["image_outlier_available_pairs"] == 3 + assert summary["image_outlier_valid_pair_counts"][-1] == 3 + assert summary["image_outlier_outlier_pair_counts"][-1] == 3 + + +def test_comparison_star_stability_summary_flags_candidate_specific_bad_frame(): + airmass = np.linspace(1.0, 1.5, 12) + comp1 = np.full(12, 100.0, dtype=float) + comp2 = np.full(12, 80.0, dtype=float) + comp3 = np.full(12, 120.0, dtype=float) + comp4 = np.full(12, 90.0, dtype=float) + comp1[7] = 60.0 + + summary = comparison_star_stability_summary( + { + "comp1": comp1, + "comp2": comp2, + "comp3": comp3, + "comp4": comp4, + }, + airmass, + ) + + assert summary["field_image_keep_mask"].all() + comp1_summary = summary["comp_summaries"][0] + comp2_summary = summary["comp_summaries"][1] + assert comp1_summary["ensemble_frame_rejected_indices"] == [7] + assert comp1_summary["ensemble_frame_rejected_count"] == 1 + assert comp1_summary["ensemble_frame_valid_pair_counts"][7] == 3 + assert comp1_summary["ensemble_frame_outlier_pair_counts"][7] == 3 + assert comp2_summary["ensemble_frame_rejected_count"] == 0 + + +def test_comparison_star_stability_summary_clips_candidate_psf_spikes_before_suitability_rejection(): + frame_count = 89 + airmass = np.linspace(1.25, 1.06, frame_count) + phase = np.linspace(0.0, 4.0 * np.pi, frame_count) + comp_flux_map = { + f"comp{index + 1}": 100.0 * (1.0 + 0.002 * np.sin(phase + index)) + for index in range(9) + } + spike_indices = np.array([2, 5, 11, 12, 13, 39, 43, 45, 56], dtype=int) + comp_flux_map["comp6"] = comp_flux_map["comp6"].copy() + comp_flux_map["comp6"][spike_indices] *= 0.35 + + summary = comparison_star_stability_summary(comp_flux_map, airmass) + comp6_summary = summary["comp_summaries"][5] + + assert comp6_summary["coverage_count"] == frame_count + assert comp6_summary["suitability_outlier_rejected"] is False + assert comp6_summary["aggregate_score"] < 0.01 + assert comp6_summary["ensemble_frame_rejected_count"] == len(spike_indices) + assert comp6_summary["ensemble_frame_rejected_indices"] == spike_indices.tolist() + + +def test_select_comparison_calibrated_photometry_masks_psf_quality_before_aperture_ensemble(): + frame_count = 30 + airmass = np.linspace(1.2, 1.0, frame_count) + + def build_psf_rows(): + rows = np.zeros((frame_count, 7), dtype=float) + rows[:, 0] = 10.0 + rows[:, 1] = 20.0 + rows[:, 2] = 200.0 + rows[:, 3] = 1.0 + rows[:, 4] = 1.0 + return rows + + psf_data = { + "target": build_psf_rows(), + "comp1": build_psf_rows(), + "comp2": build_psf_rows(), + "comp3": build_psf_rows(), + } + psf_data["comp1"][7, 2] = 40.0 + psf_data["comp1"][7, 3:5] = 6.0 + + aper_data = { + "target": np.full((frame_count, 1, 1), 1000.0), + "target_bg": np.full((frame_count, 1, 1), 10.0), + } + for key in ("comp1", "comp2", "comp3"): + aper_data[key] = np.full((frame_count, 1, 1), 100.0) + aper_data[f"{key}_bg"] = np.full((frame_count, 1, 1), 10.0) + aper_data["comp1"][7, 0, 0] = 1.0 + + calibration = select_comparison_calibrated_photometry( + psf_data, + aper_data, + apers=np.array([2.5]), + annuli=np.array([10.0]), + airmass=airmass, + comp_stars=[[10.0, 20.0], [30.0, 40.0], [50.0, 60.0]], + sigma=1.0, + use_psf_photometry=False, + use_aperture_photometry=True, + ) + comp1_summary = calibration["comp_summaries"][0] + + assert comp1_summary["psf_quality_rejected_count"] == 1 + assert comp1_summary["coverage_count"] == frame_count - 1 + assert comp1_summary["ensemble_frame_rejected_count"] == 0 + assert np.isnan(comp1_summary["ensemble_ratio_series"][7]) + + +def test_exact_comparison_calibration_does_not_reject_an_infinite_stability_score(): + frame_count = 6 + psf_rows = np.ones((frame_count, 7), dtype=float) + psf_rows[:, 3:5] = 1.0 + calibration = select_comparison_calibrated_photometry( + { + "target": psf_rows.copy(), + "comp1": psf_rows.copy(), + }, + { + "target": np.full((frame_count, 1, 1), 1000.0), + "comp1": np.full((frame_count, 1, 1), np.nan), + }, + apers=np.array([2.5]), + annuli=np.array([10.0]), + airmass=np.linspace(1.0, 1.5, frame_count), + comp_stars=[[10.0, 20.0]], + sigma=1.0, + use_psf_photometry=False, + use_aperture_photometry=True, + use_exactly_the_comps_provided=True, + ) + + assert calibration is not None + assert calibration["best_comp_index"] == 0 + assert calibration["comp_summaries"][0]["coverage_rejected"] is False + + +def test_select_comparison_calibrated_photometry_masks_overexposed_comp_measurements(): + frame_count = 24 + airmass = np.linspace(1.2, 1.0, frame_count) + + def build_psf_rows(): + rows = np.zeros((frame_count, 7), dtype=float) + rows[:, 0] = 10.0 + rows[:, 1] = 20.0 + rows[:, 2] = 200.0 + rows[:, 3] = 1.0 + rows[:, 4] = 1.0 + return rows + + psf_data = { + "target": build_psf_rows(), + "comp1": build_psf_rows(), + "comp2": build_psf_rows(), + } + aper_data = { + "target": np.full((frame_count, 1, 1), 1000.0), + "target_bg": np.full((frame_count, 1, 1), 10.0), + } + for key in ("comp1", "comp2"): + aper_data[key] = np.full((frame_count, 1, 1), 100.0) + aper_data[f"{key}_bg"] = np.full((frame_count, 1, 1), 10.0) + + comp_overexposed_masks = { + "comp1": np.zeros(frame_count, dtype=bool), + "comp2": np.zeros(frame_count, dtype=bool), + } + comp_overexposed_masks["comp2"][5] = True + aper_data["comp2"][5, 0, 0] = np.nan + + calibration = select_comparison_calibrated_photometry( + psf_data, + aper_data, + apers=np.array([2.5]), + annuli=np.array([10.0]), + airmass=airmass, + comp_stars=[[10.0, 20.0], [30.0, 40.0]], + sigma=1.0, + use_psf_photometry=False, + use_aperture_photometry=True, + comp_overexposed_masks=comp_overexposed_masks, + ) + comp2_summary = calibration["comp_summaries"][1] + + assert comp2_summary["overexposure_rejected_count"] == 1 + assert comp2_summary["coverage_count"] == frame_count - 1 + assert np.isnan(comp2_summary["ensemble_ratio_series"][5]) + assert calibration["best_comp_index"] == 0 + assert calibration["comp_summaries"][0]["coverage_count"] == frame_count + + +def test_cheap_lightcurve_prescore_treats_large_ratio_flag_as_noop(): + tflux = np.array([2.0, 2.0, 2.0, 6.0, 2.0, 2.0]) + cflux = np.full(tflux.shape[0], 2.0) + airmass = np.linspace(1.0, 1.5, tflux.shape[0]) + + score_with_flag = cheap_lightcurve_prescore(tflux, cflux, airmass, enforce_relative_flux_max=True) + score_without_flag = cheap_lightcurve_prescore(tflux, cflux, airmass, enforce_relative_flux_max=False) + + assert np.isfinite(score_with_flag) + assert np.isclose(score_with_flag, score_without_flag) + + +def test_normalize_flux_series_to_approximate_unity_scales_by_robust_baseline(): + flux = np.array([3.0, 3.3, 2.7, 3.0, 30.0], dtype=float) + unc = np.full(flux.shape[0], 0.3, dtype=float) + + normalized_flux, normalized_unc, baseline = normalize_flux_series_to_approximate_unity(flux, unc) + + assert baseline == pytest.approx(3.0) + assert np.nanmedian(normalized_flux[:4]) == pytest.approx(1.0) + assert np.nanmedian(normalized_unc[:4]) == pytest.approx(0.1) + + +def test_cheap_lightcurve_prescore_allows_large_raw_target_reference_ratios(): + tflux = np.full(6, 30.0) + cflux = np.full(6, 10.0) + airmass = np.linspace(1.0, 1.5, 6) + + score = cheap_lightcurve_prescore(tflux, cflux, airmass, enforce_relative_flux_max=False) + + assert np.isfinite(score) + + +def test_cheap_lightcurve_prescore_keeps_target_only_mode_unfiltered(): + tflux = np.array([10.0, 11.0, 12.0, 13.0, 14.0, 15.0]) + cflux = np.ones(tflux.shape[0]) + airmass = np.linspace(1.0, 1.5, tflux.shape[0]) + + score = cheap_lightcurve_prescore(tflux, cflux, airmass) + + assert np.isfinite(score) + + +def test_should_skip_airmass_fit_when_airmass_span_is_small(): + airmass = np.array([1.10, 1.12, 1.14, 1.15]) + + assert should_skip_airmass_fit(airmass) + + +def test_detrend_flux_on_out_of_transit_baseline_removes_linear_slope(): + times = np.array([-2.0, -1.0, -0.25, 0.0, 0.25, 1.0, 2.0]) + baseline = 1.0 + 0.02 * times + transit_profile = np.array([1.0, 1.0, 1.0, 0.99, 1.0, 1.0, 1.0]) + flux = baseline * transit_profile + fluxerr = np.full_like(times, 0.01) + fit = types.SimpleNamespace( + transit=transit_profile, + parameters={"tmid": 0.0}, + ) + + result = detrend_flux_on_out_of_transit_baseline(times, flux, fluxerr, fit, min_side_points=2) + + assert result["applied"] is True + assert np.allclose(result["flux"][[0, 1, 2, 4, 5, 6]], 1.0, atol=1e-8) + assert result["flux"][3] == pytest.approx(0.99, abs=1e-8) + assert result["slope"] == pytest.approx(0.02, abs=1e-8) + assert result["reference_time_bjd_tdb"] == pytest.approx(0.0) + + +def test_fit_final_lightcurve_with_oot_baseline_detrending_refits_with_flattened_flux(monkeypatch): + import exotic.exotic as exotic_module + + times = np.array([-2.0, -1.0, -0.25, 0.0, 0.25, 1.0, 2.0]) + flux = (1.0 + 0.02 * times) * np.array([1.0, 1.0, 1.0, 0.99, 1.0, 1.0, 1.0]) + fluxerr = np.full_like(times, 0.01) + airmass = np.ones_like(times) + prior = {"rprs": 0.1, "tmid": 0.0, "inc": 89.0, "a2": 0.0} + bounds = {"rprs": [0.0, 0.2], "tmid": [-0.1, 0.1], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]} + + captured = {"calls": []} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + captured["calls"].append(np.array(call_flux, dtype=float)) + return types.SimpleNamespace( + transit=np.array([1.0, 1.0, 1.0, 0.99, 1.0, 1.0, 1.0]), + parameters={"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a2": 0.01}, + data=np.array(call_flux, dtype=float), + residuals=np.zeros_like(call_flux, dtype=float), + ) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit, refit_flux, refit_unc = fit_final_lightcurve_with_oot_baseline_detrending( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + skip_airmass_fit=False, + disable_vertical_flux_normalization=False, + detrend_on_outoftransit_baseline=True, + oot_baseline_min_points_per_side=2, + ) + + assert len(captured["calls"]) == 2 + assert np.allclose(captured["calls"][0], flux) + assert np.allclose(captured["calls"][1][[0, 1, 2, 4, 5, 6]], 1.0, atol=1e-8) + assert refit_flux[3] == pytest.approx(0.99, abs=1e-8) + assert np.allclose(refit_unc[[0, 1, 2, 4, 5, 6]], 0.01 / (1.0 + 0.02 * times[[0, 1, 2, 4, 5, 6]])) + assert fit.oot_baseline_detrending_applied is True + assert fit.oot_baseline_reference_time_bjd_tdb == pytest.approx(0.0) + assert fit.oot_baseline_pre_points == 3 + assert fit.oot_baseline_post_points == 3 + + +def test_fit_final_lightcurve_linear_detrend_does_not_reapply_fixed_airmass_baseline(monkeypatch): + import exotic.exotic as exotic_module + + times = np.array([-2.0, -1.0, -0.25, 0.0, 0.25, 1.0, 2.0]) + transit_profile = np.array([1.0, 1.0, 1.0, 0.99, 1.0, 1.0, 1.0]) + flux = (1.03 + 0.02 * times) * transit_profile + fluxerr = np.full_like(times, 0.01) + airmass = np.linspace(1.0, 1.3, times.size) + prior = {"rprs": 0.1, "tmid": 0.0, "inc": 89.0, "a0": 1.03, "a1": 1.03, "a2": 0.2} + bounds = { + "rprs": [0.0, 0.2], + "tmid": [-0.1, 0.1], + "inc": [84.0, 90.0], + "a0": [0.95, 1.05], + "a2": [-3.0, 3.0], + } + captured = {"calls": []} + + def fake_run_nested( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + **kwargs, + ): + captured["calls"].append({ + "flux": np.asarray(call_flux, dtype=float), + "prior": dict(call_prior), + "bounds": dict(call_bounds), + "fixed_flux_baseline": kwargs.get("fixed_flux_baseline"), + "fixed_parameter_errors": dict(kwargs.get("fixed_parameter_errors", {})), + }) + return types.SimpleNamespace( + time=np.asarray(call_times, dtype=float), + data=np.asarray(call_flux, dtype=float), + dataerr=np.asarray(call_fluxerr, dtype=float), + airmass=np.asarray(call_airmass, dtype=float), + transit=transit_profile.copy(), + parameters={ + "tmid": 0.0, + "rprs": 0.1, + "inc": 89.0, + "a0": call_prior.get("a0", 1.0), + "a2": call_prior.get("a2", 0.2), + }, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.001, "a2": 0.01}, + residuals=np.zeros_like(call_flux, dtype=float), + duration_expected=0.5, + duration_measured=0.5, + ) + + monkeypatch.setattr(exotic_module, "run_nested_lightcurve_fit_with_rprs_posterior_retry", fake_run_nested) + monkeypatch.setattr( + exotic_module, + "build_final_fit_prefit_refinement_plan", + lambda call_times, call_flux, call_fluxerr, call_airmass, call_prior, call_bounds, fit, **kwargs: { + "applied": False, + "note": "test no prefit refinement", + "times": np.asarray(call_times, dtype=float), + "flux": np.asarray(call_flux, dtype=float), + "unc": np.asarray(call_fluxerr, dtype=float), + "airmass": np.asarray(call_airmass, dtype=float), + "jd_times": None, + "prior": dict(call_prior), + "bounds": dict(call_bounds), + "duration": 0.5, + "original_point_count": len(call_times), + "refined_point_count": len(call_times), + "trimmed_pre_points": 0, + "trimmed_post_points": 0, + "original_tmid_bounds": call_bounds["tmid"], + "refined_tmid_bounds": call_bounds["tmid"], + }, + ) + monkeypatch.setattr(exotic_module, "annotate_transit_detection_qc", lambda fit: None) + + fit, refit_flux, _ = fit_final_lightcurve_with_oot_baseline_detrending( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + detrend_on_outoftransit_baseline=True, + oot_baseline_min_points_per_side=2, + extend_sparse_posterior_live_points=False, + ) + + assert len(captured["calls"]) == 2 + final_call = captured["calls"][1] + assert final_call["fixed_flux_baseline"] is True + assert final_call["prior"]["a0"] == pytest.approx(1.0) + assert final_call["prior"]["a2"] == pytest.approx(0.0) + assert final_call["fixed_parameter_errors"]["a0"] > 0 + assert final_call["fixed_parameter_errors"]["a1"] == pytest.approx( + final_call["fixed_parameter_errors"]["a0"], + ) + assert final_call["fixed_parameter_errors"]["a2"] == pytest.approx(0.0) + assert "a0" not in final_call["bounds"] + assert "a2" not in final_call["bounds"] + assert np.allclose(final_call["flux"][[0, 1, 2, 4, 5, 6]], 1.0, atol=1e-8) + assert final_call["flux"][3] == pytest.approx(0.99, abs=1e-8) + assert np.allclose(refit_flux, final_call["flux"]) + assert fit.oot_baseline_parameter_fit_applied is False + assert "already flattened" in fit.oot_baseline_parameter_fit_note + assert np.isfinite(fit.oot_baseline_parameter_fit_a0) + assert fit.oot_baseline_parameter_fit_a0_error > 0 + assert np.isfinite(fit.oot_baseline_parameter_fit_a2) + assert fit.oot_baseline_parameter_fit_a2_error > 0 + assert fit.pre_detrending_baseline_source.startswith("out-of-transit airmass/baseline") + assert fit.pre_detrending_baseline_scale_parameter == "a0" + assert fit.pre_detrending_baseline_scale_value == pytest.approx( + fit.oot_baseline_parameter_fit_a0 + ) + assert fit.pre_detrending_baseline_scale_error == pytest.approx( + fit.oot_baseline_parameter_fit_a0_error + ) + assert fit.pre_detrending_baseline_a2_value == pytest.approx( + fit.oot_baseline_parameter_fit_a2 + ) + assert fit.pre_detrending_baseline_a2_error == pytest.approx( + fit.oot_baseline_parameter_fit_a2_error + ) + + +def test_fit_final_lightcurve_uses_oot_baseline_parameter_refit_when_linear_detrend_skips(monkeypatch): + import exotic.exotic as exotic_module + + times = np.array([-0.03, -0.02, -0.01, 0.00, 0.01, 0.02, 0.03]) + transit_profile = np.array([0.99, 0.99, 0.99, 0.99, 1.0, 1.0, 1.0]) + airmass = np.linspace(1.0, 1.6, times.size) + flux = np.exp(0.2 * (airmass - np.mean(airmass))) * transit_profile + fluxerr = np.full_like(times, 0.01) + prior = {"rprs": 0.1, "tmid": -0.01, "inc": 89.0, "a2": 0.0} + bounds = { + "rprs": [0.0, 0.2], + "tmid": [-0.03, 0.01], + "inc": [84.0, 90.0], + "a0": [0.95, 1.05], + "a2": [-3.0, 3.0], + } + captured = {"calls": []} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + baseline_fit_mask=None, + fixed_parameter_errors=None, + ): + captured["calls"].append({ + "bounds": dict(call_bounds), + "baseline_fit_mask": None if baseline_fit_mask is None else np.asarray(baseline_fit_mask, dtype=bool), + "fixed_parameter_errors": dict(fixed_parameter_errors or {}), + "prior": dict(call_prior), + }) + return types.SimpleNamespace( + transit=transit_profile.copy(), + parameters={ + "tmid": -0.01, + "rprs": 0.1, + "inc": 89.0, + "a2": call_prior.get("a2", 0.0), + "a0": call_prior.get("a0", 1.0), + "a1": call_prior.get("a0", 1.0), + }, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a2": 0.01, "a0": 0.001, "a1": 0.001}, + data=np.array(call_flux, dtype=float), + residuals=np.zeros_like(call_flux, dtype=float), + ) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit, _, _ = fit_final_lightcurve_with_oot_baseline_detrending( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + detrend_on_outoftransit_baseline=True, + oot_baseline_min_points_per_side=0, + ) + + assert len(captured["calls"]) == 2 + assert captured["calls"][0]["baseline_fit_mask"] is None + assert captured["calls"][1]["baseline_fit_mask"].tolist() == [False, False, False, False, True, True, True] + assert "a0" not in captured["calls"][1]["bounds"] + assert "a2" not in captured["calls"][1]["bounds"] + assert captured["calls"][1]["fixed_parameter_errors"]["a0"] > 0 + assert captured["calls"][1]["fixed_parameter_errors"]["a1"] == pytest.approx( + captured["calls"][1]["fixed_parameter_errors"]["a0"], + ) + assert "a2" in captured["calls"][1]["fixed_parameter_errors"] + assert fit.oot_baseline_parameter_fit_applied is True + assert fit.oot_baseline_detrending_applied is False + + +def test_phase_bin_sigma_clip_flags_local_phase_outlier(): + phase_centers = np.linspace(-0.045, 0.045, 10) + phase = np.concatenate([center + np.linspace(-1e-4, 1e-4, 5) for center in phase_centers]) + base_profile = np.array([-0.002, -0.001, 0.0, 0.001, 0.002]) + values = np.concatenate([1.0 + base_profile for _ in phase_centers]) + values[27] = 1.15 + + mask = phase_bin_sigma_clip(values, phase, sigma=3, bins=10) + + assert mask.sum() == 1 + assert mask[27] + + +def test_sigma_clip_respects_large_time_gaps_between_segments(): + times_pre = 2461151.80 + np.arange(50, dtype=float) * 0.00075 + times_post = 2461151.98 + np.arange(8, dtype=float) * 0.00075 + times = np.concatenate([times_pre, times_post]) + + rng = np.random.default_rng(42) + values_pre = ( + 0.0235 + + 0.0006 * np.sin(np.linspace(0, 8 * np.pi, times_pre.size)) + + 0.0004 * np.linspace(0, 1, times_pre.size) + + rng.normal(0, 5e-5, times_pre.size) + ) + values_post = np.full(times_post.size, np.nanmedian(values_pre[-20:]) - 8e-4) + values_post += rng.normal(0, 5e-5, times_post.size) + values = np.concatenate([values_pre, values_post]) + values[54] += 0.8 + + np.random.seed(0) + old_mask = sigma_clip(values, sigma=3, dt=37, times=None) + np.random.seed(0) + gap_aware_mask = sigma_clip(values, sigma=3, dt=37, times=times) + + assert old_mask[54:58].all() + assert not gap_aware_mask[54:58].any() + assert gap_aware_mask.sum() < old_mask.sum() + + +def test_fit_final_lightcurve_preserves_explicit_plot_time_range(monkeypatch): + import exotic.exotic as exotic_module + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + return types.SimpleNamespace( + parameters={"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a2": 0.01}, + data=np.array(call_flux, dtype=float), + residuals=np.zeros_like(call_flux, dtype=float), + ) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.linspace(0.0, 0.05, 6) + flux = np.ones(6, dtype=float) + fluxerr = np.full(6, 0.01, dtype=float) + airmass = np.linspace(1.0, 1.5, 6) + prior = {"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0} + bounds = {"rprs": [0.05, 0.15], "tmid": [-0.01, 0.01], "inc": [84.0, 90.0]} + plot_time_range = (-0.12, 0.18) + + fit, _, _ = fit_final_lightcurve_with_oot_baseline_detrending( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + detrend_on_outoftransit_baseline=False, + plot_time_range=plot_time_range, + ) + + assert fit.plot_time_range == pytest.approx(plot_time_range) + + +def test_fit_final_lightcurve_retries_nested_fit_when_rprs_posterior_is_clipped(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_ENABLED", False) + captured = {"calls": []} + + def make_fit(call_flux, diagnostics): + fit = types.SimpleNamespace( + parameters={"tmid": 0.0, "rprs": 0.152, "inc": 89.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.002, "inc": 0.1, "a2": 0.01}, + data=np.array(call_flux, dtype=float), + residuals=np.zeros_like(call_flux, dtype=float), + ) + fit.get_parameter_posterior_recenter_diagnostics = lambda key: diagnostics if key == "rprs" else None + return fit + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + captured["calls"].append({ + "prior": dict(call_prior), + "bounds": {key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value for key, value in call_bounds.items()}, + }) + if len(captured["calls"]) == 1: + return make_fit( + call_flux, + { + "clipped": True, + "edge": "upper", + "mode": 0.158, + "std": 0.006, + "bounds": [0.128, 0.188], + "reason": "posterior peaks against the upper search bound.", + }, + ) + return make_fit( + call_flux, + { + "clipped": False, + "edge": None, + "mode": 0.159, + "std": 0.005, + "bounds": [0.128, 0.188], + "reason": "posterior support is comfortably inside the sampled bounds.", + }, + ) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.linspace(-0.03, 0.03, 7) + flux = np.ones(7, dtype=float) + fluxerr = np.full(7, 0.01, dtype=float) + airmass = np.ones(7, dtype=float) + prior = {"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0} + bounds = {"rprs": [0.0, 0.125], "tmid": [-0.01, 0.01], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]} + + fit, _, _ = fit_final_lightcurve_with_oot_baseline_detrending( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + detrend_on_outoftransit_baseline=False, + ) + + assert len(captured["calls"]) == 2 + assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([0.0, 0.125]) + assert captured["calls"][1]["prior"]["rprs"] == pytest.approx(0.158) + assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.0, 0.208]) + assert fit.rprs_posterior_refit_applied is True + assert fit.rprs_posterior_refit_count == 1 + assert fit.rprs_posterior_refit_edge == "upper" + assert fit.rprs_posterior_refit_bounds == pytest.approx([0.0, 0.208]) + + +def test_fit_final_lightcurve_carries_retry_bounds_into_oot_baseline_refit(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_ENABLED", False) + times = np.array([-2.0, -1.0, -0.25, 0.0, 0.25, 1.0, 2.0]) + flux = (1.0 + 0.02 * times) * np.array([1.0, 1.0, 1.0, 0.99, 1.0, 1.0, 1.0]) + fluxerr = np.full_like(times, 0.01) + airmass = np.ones_like(times) + prior = {"rprs": 0.1, "tmid": 0.0, "inc": 89.0, "a2": 0.0} + bounds = {"rprs": [0.0, 0.125], "tmid": [-0.1, 0.1], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]} + transit_model = np.array([1.0, 1.0, 1.0, 0.99, 1.0, 1.0, 1.0]) + diagnostics = [ + { + "clipped": True, + "edge": "upper", + "mode": 0.158, + "std": 0.006, + "bounds": [0.128, 0.188], + "reason": "posterior peaks against the upper search bound.", + }, + { + "clipped": False, + "edge": None, + "mode": 0.159, + "std": 0.005, + "bounds": [0.108, 0.208], + "reason": "posterior support is comfortably inside the sampled bounds.", + }, + { + "clipped": False, + "edge": None, + "mode": 0.160, + "std": 0.005, + "bounds": [0.108, 0.208], + "reason": "posterior support is comfortably inside the sampled bounds.", + }, + ] + captured = {"calls": []} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + call_index = len(captured["calls"]) + call_diagnostics = diagnostics[min(call_index, len(diagnostics) - 1)] + captured["calls"].append({ + "flux": np.array(call_flux, dtype=float), + "prior": dict(call_prior), + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + fit = types.SimpleNamespace( + transit=transit_model, + parameters={"tmid": 0.0, "rprs": call_diagnostics["mode"], "inc": 89.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a2": 0.01}, + data=np.array(call_flux, dtype=float), + residuals=np.zeros_like(call_flux, dtype=float), + ) + fit.get_parameter_posterior_recenter_diagnostics = ( + lambda key: dict(call_diagnostics) if key == "rprs" else None + ) + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit, _, _ = fit_final_lightcurve_with_oot_baseline_detrending( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + detrend_on_outoftransit_baseline=True, + oot_baseline_min_points_per_side=2, + ) + + assert len(captured["calls"]) == 3 + assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([0.0, 0.125]) + assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.0, 0.208]) + assert captured["calls"][2]["bounds"]["rprs"] == pytest.approx([0.0, 0.208]) + assert np.allclose(captured["calls"][2]["flux"][[0, 1, 2, 4, 5, 6]], 1.0, atol=1e-8) + assert fit.oot_baseline_detrending_applied is True + + +def test_fit_final_lightcurve_prefit_refinement_trims_baseline_and_recenters_tmid(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + captured["calls"].append({ + "times": np.array(call_times, dtype=float), + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + return types.SimpleNamespace( + duration_expected=2.0, + duration_measured=2.0, + parameters={"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0, "per": 10.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a2": 0.01}, + data=np.array(call_flux, dtype=float), + residuals=np.zeros_like(call_flux, dtype=float), + ) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.array([-3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0], dtype=float) + flux = np.ones(times.shape[0], dtype=float) + fluxerr = np.full(times.shape[0], 0.01, dtype=float) + airmass = np.ones(times.shape[0], dtype=float) + prior = {"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0, "per": 10.0} + bounds = {"rprs": [0.0, 0.2], "tmid": [-2.0, 2.0], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]} + + fit, trimmed_flux, trimmed_unc = fit_final_lightcurve_with_oot_baseline_detrending( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + detrend_on_outoftransit_baseline=False, + baseline_duration_multiplier=0.5, + ) + + assert len(captured["calls"]) == 2 + assert captured["calls"][0]["times"] == pytest.approx(times) + assert captured["calls"][1]["times"] == pytest.approx(np.array([-2.0, -1.0, 0.0, 1.0, 2.0])) + assert captured["calls"][1]["bounds"]["tmid"] == pytest.approx([-1.0, 1.0]) + assert trimmed_flux == pytest.approx(np.ones(5)) + assert trimmed_unc == pytest.approx(np.full(5, 0.01)) + assert fit.prefit_refinement_applied is True + assert fit.prefit_refinement_trimmed_pre_points == 1 + assert fit.prefit_refinement_trimmed_post_points == 1 + assert fit.prefit_refinement_tmid_bounds == pytest.approx([-1.0, 1.0]) + + +def test_fit_final_lightcurve_prefit_refinement_skips_one_sided_transit_solution(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + call_times = np.array(call_times, dtype=float) + captured["calls"].append({ + "times": call_times, + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + transit = np.ones_like(call_times, dtype=float) + transit[call_times >= 0.0] = 0.95 + return types.SimpleNamespace( + duration_expected=2.0, + duration_measured=2.0, + parameters={"tmid": 2.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0, "per": 10.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a2": 0.01}, + data=np.array(call_flux, dtype=float), + residuals=np.zeros_like(call_flux, dtype=float), + time=call_times, + transit=transit, + ) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.array([-3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0], dtype=float) + flux = np.ones(times.shape[0], dtype=float) + fluxerr = np.full(times.shape[0], 0.01, dtype=float) + airmass = np.ones(times.shape[0], dtype=float) + prior = {"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0, "per": 10.0} + bounds = {"rprs": [0.0, 0.2], "tmid": [-2.0, 2.0], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]} + + fit, trimmed_flux, trimmed_unc = fit_final_lightcurve_with_oot_baseline_detrending( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + detrend_on_outoftransit_baseline=False, + baseline_duration_multiplier=0.5, + ) + + assert len(captured["calls"]) == 1 + assert captured["calls"][0]["times"] == pytest.approx(times) + assert trimmed_flux == pytest.approx(flux) + assert trimmed_unc == pytest.approx(fluxerr) + assert fit.prefit_refinement_applied is False + assert "one side of the modeled transit" in fit.prefit_refinement_note + + +def test_fit_final_lightcurve_prefit_refinement_does_not_expand_tmid_past_original_bounds(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + call_times = np.array(call_times, dtype=float) + captured["calls"].append({ + "times": call_times, + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + transit = np.where(np.abs(call_times - 0.4) <= 0.25, 0.98, 1.0) + return types.SimpleNamespace( + duration_expected=0.5, + duration_measured=0.5, + parameters={"tmid": 0.4, "rprs": 0.1, "inc": 89.0, "a2": 0.0, "per": 10.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a2": 0.01}, + data=np.array(call_flux, dtype=float), + residuals=np.zeros_like(call_flux, dtype=float), + time=call_times, + transit=transit, + ) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.array([-0.8, -0.4, 0.0, 0.4, 0.8], dtype=float) + flux = np.ones(times.shape[0], dtype=float) + fluxerr = np.full(times.shape[0], 0.01, dtype=float) + airmass = np.ones(times.shape[0], dtype=float) + prior = {"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0, "per": 10.0} + bounds = {"rprs": [0.0, 0.2], "tmid": [-0.5, 0.5], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]} + + fit, _, _ = fit_final_lightcurve_with_oot_baseline_detrending( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + detrend_on_outoftransit_baseline=False, + baseline_duration_multiplier=0.5, + ) + + assert len(captured["calls"]) == 2 + assert captured["calls"][1]["bounds"]["tmid"] == pytest.approx([0.15, 0.5]) + assert fit.prefit_refinement_tmid_bounds == pytest.approx([0.15, 0.5]) + + +def test_fit_lightcurve_keeps_large_raw_target_reference_ratios(monkeypatch): + captured = {} + + def fake_lc_fitter( + times, + fluxes, + flux_unc, + airmass, + prior, + bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + captured["times"] = np.array(times) + captured["fluxes"] = np.array(fluxes) + captured["flux_unc"] = np.array(flux_unc) + captured["airmass"] = np.array(airmass) + captured["jd_times"] = np.array(jd_times) + captured["mode"] = mode + return types.SimpleNamespace() + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 0.05, 6) + tflux = np.array([2.0, 2.0, 2.0, 6.0, 2.0, 2.0]) + cflux = np.full(tflux.shape[0], 2.0) + airmass = np.linspace(1.0, 1.5, tflux.shape[0]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + + myfit, fit_tflux, fit_cflux = fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times) + + assert myfit is not None + assert captured["mode"] == "lm" + assert len(captured["fluxes"]) == 6 + assert np.allclose(captured["fluxes"], np.array([1.0, 1.0, 1.0, 3.0, 1.0, 1.0])) + assert np.allclose(fit_tflux, tflux) + assert np.allclose(fit_cflux, 2.0) + + +def test_fit_lightcurve_preserves_explicit_plot_time_range(monkeypatch): + captured = {} + + def fake_lc_fitter( + times, + fluxes, + flux_unc, + airmass, + prior, + bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + fit = types.SimpleNamespace() + captured["fit"] = fit + return fit + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 0.05, 6) + tflux = np.full(times.shape[0], 2.0) + cflux = np.full(times.shape[0], 2.0) + airmass = np.linspace(1.0, 1.5, times.shape[0]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + plot_time_range = (-0.12, 0.18) + + myfit, _, _ = fit_lightcurve( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times, + plot_time_range=plot_time_range, + ) + + assert myfit is captured["fit"] + assert myfit.plot_time_range == pytest.approx(plot_time_range) + + +def test_fit_lightcurve_centers_vertical_flux_bound_on_normalized_flux(monkeypatch): + captured = {} + + def fake_lc_fitter( + times, + fluxes, + flux_unc, + airmass, + prior, + bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + fit = types.SimpleNamespace() + captured["fit"] = fit + captured["prior"] = dict(prior) + captured["bounds"] = { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in bounds.items() + } + return fit + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 0.05, 6) + tflux = np.full(times.shape[0], 100.0) + cflux = np.full(times.shape[0], 2000.0) + airmass = np.linspace(1.0, 1.5, times.shape[0]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + + myfit, _, _ = fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times) + + assert myfit is captured["fit"] + assert captured["prior"]["a0"] == pytest.approx(1.0) + assert captured["prior"]["a1"] == pytest.approx(1.0) + assert captured["bounds"]["a0"] == pytest.approx([0.95, 1.05]) + + +def test_fit_lightcurve_rejects_undersampled_series(monkeypatch): + called = {"count": 0} + + def fake_lc_fitter(*args, **kwargs): + called["count"] += 1 + return types.SimpleNamespace() + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 0.03, 4) + tflux = np.full(times.shape[0], 2.0) + cflux = np.full(times.shape[0], 2.0) + airmass = np.linspace(1.0, 1.3, times.shape[0]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + + myfit, fit_tflux, fit_cflux = fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times) + + assert myfit is None + assert fit_tflux is None + assert fit_cflux is None + assert called["count"] == 0 + + +def test_fit_lightcurve_refits_after_phase_binned_clip(monkeypatch): + captured_calls = [] + + def fake_lc_fitter( + times, + fluxes, + flux_unc, + airmass, + prior, + bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + call_index = len(captured_calls) + captured_calls.append({ + "times": np.array(times), + "fluxes": np.array(fluxes), + "flux_unc": np.array(flux_unc), + "airmass": np.array(airmass), + "jd_times": np.array(jd_times), + "mode": mode, + }) + if call_index == 0: + return types.SimpleNamespace( + residuals=np.zeros(len(times)), + phase=np.linspace(-0.05, 0.05, len(times)), + ) + return types.SimpleNamespace() + + def fake_phase_bin_sigma_clip(values, phase, sigma=3, bins=10, min_points=5, max_iters=3): + mask = np.zeros(len(values), dtype=bool) + mask[-1] = True + return mask + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + monkeypatch.setattr("exotic.exotic.phase_bin_sigma_clip", fake_phase_bin_sigma_clip) + + times = np.linspace(0.0, 0.08, 8) + tflux = np.full(times.shape[0], 2.0) + cflux = np.full(times.shape[0], 2.0) + airmass = np.linspace(1.0, 1.5, times.shape[0]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + + myfit, fit_tflux, fit_cflux = fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times) + + assert myfit is not None + assert len(captured_calls) == 2 + assert captured_calls[0]["mode"] == "lm" + assert captured_calls[1]["mode"] == "lm" + assert len(captured_calls[0]["times"]) == 8 + assert len(captured_calls[1]["times"]) == 7 + assert len(fit_tflux) == 7 + assert len(fit_cflux) == 7 + + +def test_fit_lightcurve_runs_nested_fit_when_requested(monkeypatch): + captured_modes = [] + captured_duration_priors = [] + + def fake_lc_fitter( + times, + fluxes, + flux_unc, + airmass, + prior, + bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + captured_modes.append(mode) + captured_duration_priors.append(duration_prior) + return types.SimpleNamespace() + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 0.05, 6) + tflux = np.full(times.shape[0], 2.0) + cflux = np.full(times.shape[0], 2.0) + airmass = np.linspace(1.0, 1.5, times.shape[0]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + + myfit, _, _ = fit_lightcurve( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times, + final_fit_mode="ns", + ) + + assert myfit is not None + assert captured_modes == ["lm", "ns"] + assert captured_duration_priors[0] is None + assert captured_duration_priors[1] is not None + assert captured_duration_priors[1]["applied"] is True + assert captured_duration_priors[1]["expected_duration"] > 0 + assert myfit.pre_ultranest_transit_coverage_valid is True + assert myfit.pre_ultranest_transit_coverage_expected_successful is True + + +def test_fit_lightcurve_attaches_frame_filter_diagnostics(monkeypatch): + def fake_lc_fitter( + times, + fluxes, + flux_unc, + airmass, + prior, + bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + return types.SimpleNamespace( + time=np.asarray(times, dtype=float), + data=np.asarray(fluxes, dtype=float), + dataerr=np.asarray(flux_unc, dtype=float), + detrended=np.asarray(fluxes, dtype=float), + detrendederr=np.asarray(flux_unc, dtype=float), + airmass=np.asarray(airmass, dtype=float), + airmass_model=np.ones(len(times), dtype=float), + residuals=np.zeros(len(times), dtype=float), + phase=np.linspace(-0.1, 0.1, len(times)), + transit=np.ones(len(times), dtype=float), + model=np.asarray(fluxes, dtype=float), + wf=np.ones(len(times), dtype=float), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a1": 1.0, "a2": 0.0, "per": 1.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a1": 0.01, "a2": 0.01}, + ) + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.array([False] * (len(data) - 1) + [True], dtype=bool), + ) + monkeypatch.setattr( + "exotic.exotic.phase_bin_sigma_clip", + lambda values, phase, sigma=3, bins=10, min_points=5, max_iters=3: np.zeros(len(values), dtype=bool), + ) + + times = np.arange(12, dtype=float) + tflux = np.full(times.shape[0], 100.0, dtype=float) + cflux = np.full(times.shape[0], 50.0, dtype=float) + cflux[1] = 0.0 + airmass = np.linspace(1.0, 1.5, times.shape[0]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.5, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + + myfit, _, _ = fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times) + + assert myfit is not None + diagnostics = myfit.frame_filter_diagnostics + assert [diagnostic["stage"] for diagnostic in diagnostics] == [ + "Target/reference ratio filter", + "Initial sigma clip", + "Pre-fit raw-ratio outlier clip", + "Finite/positive photometry filter", + ] + assert diagnostics[0]["dropped_point_count"] == 1 + assert diagnostics[0]["first_dropped_time"] == pytest.approx(1.0) + assert diagnostics[1]["dropped_point_count"] == 1 + assert diagnostics[1]["first_dropped_time"] == pytest.approx(11.0) + assert diagnostics[2]["dropped_point_count"] == 0 + assert diagnostics[3]["dropped_point_count"] == 0 + + +def test_evaluate_transit_detection_qc_prefers_transit_model(): + transit_model = np.ones(21, dtype=float) + transit_model[8:13] = 0.99 + data = transit_model + np.array( + [ + 0.0002, -0.0001, 0.0001, -0.0002, 0.0000, 0.0001, -0.0001, + 0.0002, -0.0002, 0.0001, -0.0001, 0.0002, -0.0002, 0.0001, + 0.0000, -0.0001, 0.0002, -0.0001, 0.0001, 0.0000, -0.0001, + ], + dtype=float, + ) + fit = types.SimpleNamespace( + data=data, + dataerr=np.full(data.shape[0], 0.0015, dtype=float), + model=transit_model, + airmass=np.ones(data.shape[0], dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.10, "tmid": 0.5, "inc": 89.0, "a2": 0.0}, + errors={"rprs": 0.01, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 1.0], "tmid": [0.4, 0.6], "inc": [80.0, 90.0]}, + duration_expected=5.0, + duration_measured=5.0, + ) + + summary = evaluate_transit_detection_qc(fit) + + assert summary["computed"] is True + assert summary["preferred_model"] == "transit" + assert summary["status"] == "pass" + assert summary["delta_bic"] > 10.0 + assert summary["delta_chi2"] > 0.0 + + +def test_evaluate_transit_detection_qc_passes_strong_model_with_low_rprs_precision(): + transit_model = np.ones(21, dtype=float) + transit_model[8:13] = 0.99 + data = transit_model + np.array( + [ + 0.0002, -0.0001, 0.0001, -0.0002, 0.0000, 0.0001, -0.0001, + 0.0002, -0.0002, 0.0001, -0.0001, 0.0002, -0.0002, 0.0001, + 0.0000, -0.0001, 0.0002, -0.0001, 0.0001, 0.0000, -0.0001, + ], + dtype=float, + ) + fit = types.SimpleNamespace( + data=data, + dataerr=np.full(data.shape[0], 0.0015, dtype=float), + model=transit_model, + airmass=np.ones(data.shape[0], dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.10, "tmid": 0.5, "inc": 89.0, "a2": 0.0}, + errors={"rprs": 0.20, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 1.0], "tmid": [0.4, 0.6], "inc": [80.0, 90.0]}, + duration_expected=5.0, + duration_measured=5.0, + ) + + summary = evaluate_transit_detection_qc(fit) + + assert summary["computed"] is True + assert summary["status"] == "pass" + assert summary["rprs_sigma"] == pytest.approx(0.5) + assert summary["ktmf_metric"] >= 3.5 + assert "not used as a transit-detection veto" in " ".join(summary["notes"]) + + +def test_evaluate_transit_detection_qc_uses_ktmf_marginal_band_despite_weak_bic(monkeypatch): + monkeypatch.setattr( + "exotic.exotic.compute_transit_qc_ktmf", + lambda summary: (3.34, []), + ) + transit_model = np.ones(21, dtype=float) + transit_model[8:13] = 0.99 + data = np.ones(21, dtype=float) + fit = types.SimpleNamespace( + data=data, + dataerr=np.full(data.shape[0], 0.02, dtype=float), + model=transit_model, + airmass=np.ones(data.shape[0], dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.10, "tmid": 0.5, "inc": 89.0, "a2": 0.0}, + errors={"rprs": 0.02, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 1.0], "tmid": [0.4, 0.6], "inc": [80.0, 90.0]}, + duration_expected=5.0, + duration_measured=5.0, + ) + + summary = evaluate_transit_detection_qc(fit) + + assert summary["computed"] is True + assert summary["delta_bic"] < 6.0 + assert summary["ktmf_metric"] == pytest.approx(3.34) + assert summary["status"] == "marginal" + assert "KTMF indicates a marginal transit fit" in summary["summary"] + + +def test_evaluate_transit_detection_qc_uses_midpoint_anchored_duration_for_partial(): + times = np.linspace(0.0, 3.0, 13) + transit_model = np.ones(times.shape[0], dtype=float) + transit_model[times <= 2.25] = 0.99 + data = transit_model + np.array( + [ + 0.0002, -0.0001, 0.0001, -0.0002, 0.0000, 0.0001, -0.0001, + 0.0002, -0.0002, 0.0001, -0.0001, 0.0002, -0.0002, + ], + dtype=float, + ) + fit = types.SimpleNamespace( + time=times, + data=data, + dataerr=np.full(data.shape[0], 0.0015, dtype=float), + transit=transit_model, + model=transit_model, + airmass=np.ones(data.shape[0], dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.10, "tmid": 0.0, "inc": 89.0, "a2": 0.0}, + errors={"rprs": 0.01, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 1.0], "tmid": [-0.1, 0.1], "inc": [80.0, 90.0]}, + duration_expected=5.0, + duration_measured=2.5, + pre_ultranest_transit_coverage={ + "valid": True, + "covers_ingress": False, + "covers_mid_transit": True, + "covers_egress": True, + "observed_segment": "mid-transit to egress", + "expected_tmid": 0.0, + }, + ) + + summary = evaluate_transit_detection_qc(fit) + contributions_by_label = { + contribution["label"]: contribution + for contribution in summary["ktmf_contributions"] + } + + assert summary["duration_measured_for_qc"] == pytest.approx(4.75) + assert summary["duration_ratio"] == pytest.approx(0.95) + assert contributions_by_label["Duration Consistency"]["available"] is True + assert "midpoint-anchored partial estimate" in contributions_by_label["Duration Consistency"]["detail"] + + +def test_evaluate_transit_detection_qc_skips_duration_for_edge_only_partial(): + times = np.linspace(-3.0, -0.25, 12) + transit_model = np.ones(times.shape[0], dtype=float) + transit_model[times >= -2.5] = 0.99 + data = transit_model + np.array( + [ + 0.0002, -0.0001, 0.0001, -0.0002, 0.0000, 0.0001, + -0.0001, 0.0002, -0.0002, 0.0001, -0.0001, 0.0002, + ], + dtype=float, + ) + fit = types.SimpleNamespace( + time=times, + data=data, + dataerr=np.full(data.shape[0], 0.0015, dtype=float), + transit=transit_model, + model=transit_model, + airmass=np.ones(data.shape[0], dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.10, "tmid": 0.0, "inc": 89.0, "a2": 0.0}, + errors={"rprs": 0.01, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 1.0], "tmid": [-0.1, 0.1], "inc": [80.0, 90.0]}, + duration_expected=5.0, + duration_measured=2.75, + pre_ultranest_transit_coverage={ + "valid": True, + "covers_ingress": True, + "covers_mid_transit": False, + "covers_egress": False, + "observed_segment": "ingress-only partial", + "expected_tmid": 0.0, + }, + ) + + summary = evaluate_transit_detection_qc(fit) + contributions_by_label = { + contribution["label"]: contribution + for contribution in summary["ktmf_contributions"] + } + + assert np.isnan(summary["duration_ratio"]) + assert summary["duration_consistency_applicable"] is False + assert contributions_by_label["Duration Consistency"]["available"] is False + assert "only partially observed" in contributions_by_label["Duration Consistency"]["detail"] + + +def test_evaluate_transit_detection_qc_fails_when_flat_model_is_better(): + transit_model = np.ones(21, dtype=float) + transit_model[8:13] = 0.99 + data = np.ones(21, dtype=float) + np.array( + [ + 0.0002, -0.0001, 0.0001, -0.0002, 0.0000, 0.0001, -0.0001, + 0.0002, -0.0002, 0.0001, -0.0001, 0.0002, -0.0002, 0.0001, + 0.0000, -0.0001, 0.0002, -0.0001, 0.0001, 0.0000, -0.0001, + ], + dtype=float, + ) + fit = types.SimpleNamespace( + data=data, + dataerr=np.full(data.shape[0], 0.0015, dtype=float), + model=transit_model, + airmass=np.ones(data.shape[0], dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.10, "tmid": 0.5, "inc": 89.0, "a2": 0.0}, + errors={"rprs": 0.01, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 1.0], "tmid": [0.4, 0.6], "inc": [80.0, 90.0]}, + duration_expected=5.0, + duration_measured=5.0, + ) + + summary = evaluate_transit_detection_qc(fit) + + assert summary["computed"] is True + assert summary["status"] == "fail" + assert summary["preferred_model"] == "flat" + assert summary["delta_chi2"] < 0.0 + + +def test_evaluate_transit_detection_qc_marks_large_expected_value_deviation_fail_via_ktmf(): + transit_model = np.ones(21, dtype=float) + transit_model[8:13] = 0.99 + data = transit_model + np.array( + [ + 0.0002, -0.0001, 0.0001, -0.0002, 0.0000, 0.0001, -0.0001, + 0.0002, -0.0002, 0.0001, -0.0001, 0.0002, -0.0002, 0.0001, + 0.0000, -0.0001, 0.0002, -0.0001, 0.0001, 0.0000, -0.0001, + ], + dtype=float, + ) + fit = types.SimpleNamespace( + data=data, + dataerr=np.full(data.shape[0], 0.0015, dtype=float), + model=transit_model, + airmass=np.ones(data.shape[0], dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.18, "tmid": 0.506, "inc": 89.0, "a2": 0.0}, + errors={"rprs": 0.01, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 1.0], "tmid": [0.4, 0.6], "inc": [80.0, 90.0]}, + duration_expected=5.0, + duration_measured=5.0, + transit_qc_expected_tmid=0.5, + transit_qc_expected_tmid_unc=0.001, + transit_qc_expected_rprs=0.10, + transit_qc_expected_rprs_unc=0.01, + transit_qc_use_deviation_from_expected_transit_in_qc=True, + transit_qc_deviation_sigma_threshold=5.0, + ) + + summary = evaluate_transit_detection_qc(fit) + + assert summary["computed"] is True + assert summary["status"] == "fail" + expected_comparison_unc = np.sqrt(0.01 ** 2 + 0.01 ** 2 + (0.05 * 0.10) ** 2) + assert summary["rprs_deviation_unc"] == pytest.approx(expected_comparison_unc) + assert summary["rprs_deviation_systematic_floor"] == pytest.approx(0.05 * 0.10) + assert summary["rprs_deviation_sigma"] == pytest.approx(abs(0.18 - 0.10) / expected_comparison_unc) + assert summary["deviation_from_expected_value"] == pytest.approx(0.0) + assert summary["ktmf_metric"] <= 5.0 + assert summary["ktmf_metric"] < 3.0 + assert np.isnan(summary["tmid_deviation_sigma"]) + assert np.isnan(summary["tmid_deviation_minutes"]) + assert summary["rprs_deviation_fit_unc"] == pytest.approx(0.01) + + +def test_annotate_transit_qc_expected_values_prefers_propagated_epoch_tmid(): + fit = types.SimpleNamespace( + initial_tmid_search_tmid=2460658.8654321, + initial_tmid_search_uncertainty=0.0025, + ) + + annotate_transit_qc_expected_values( + fit, + { + "midT": 2455867.402743, + "midTUnc": 4.9e-05, + "rprs": 0.1488, + "rprsUnc": 0.00055, + }, + ) + + assert fit.transit_qc_expected_tmid == pytest.approx(2460658.8654321) + assert fit.transit_qc_expected_tmid_unc == pytest.approx(0.0025) + assert fit.transit_qc_expected_rprs == pytest.approx(0.1488) + assert fit.transit_qc_expected_rprs_unc == pytest.approx(0.00055) + + +def test_annotate_transit_qc_expected_values_coerces_scalar_like_inputs(): + fit = types.SimpleNamespace( + initial_tmid_search_tmid=np.array(["2460658.8654321"]), + initial_tmid_search_uncertainty="0.0025", + ) + + annotate_transit_qc_expected_values( + fit, + { + "midT": "2455867.402743", + "midTUnc": ["4.9e-05"], + "rprs": "0.1488", + "rprsUnc": np.array(["0.00055"]), + "use_deviation_from_expected_transit_in_qc": "n", + "deviation_from_expected_transit_in_qc_sigma": "7.5", + }, + ) + + assert fit.transit_qc_expected_tmid == pytest.approx(2460658.8654321) + assert fit.transit_qc_expected_tmid_unc == pytest.approx(0.0025) + assert fit.transit_qc_expected_rprs == pytest.approx(0.1488) + assert fit.transit_qc_expected_rprs_unc == pytest.approx(0.00055) + assert fit.transit_qc_use_deviation_from_expected_transit_in_qc is False + assert fit.transit_qc_deviation_sigma_threshold == pytest.approx(7.5) + + +def test_evaluate_transit_detection_qc_does_not_calculate_tmid_expected_value_deviation(): + times = np.linspace(0.0, 1.0, 21) + transit_model = np.ones(times.shape[0], dtype=float) + transit_model[9:12] -= 0.02 + data = transit_model + np.array( + [ + 0.0001, -0.0001, 0.0002, -0.0002, 0.0000, 0.0001, -0.0001, + 0.0002, -0.0002, 0.0001, 0.0000, -0.0001, 0.0002, -0.0002, + 0.0001, 0.0000, -0.0001, 0.0001, -0.0001, 0.0000, 0.0001, + ], + dtype=float, + ) + fit = types.SimpleNamespace( + time=times, + data=data, + dataerr=np.full(data.shape[0], 0.0015, dtype=float), + model=transit_model, + airmass=np.ones(data.shape[0], dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.10, "tmid": 0.528, "inc": 89.0, "a2": 0.0, "per": 2.0}, + errors={"rprs": 0.01, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 1.0], "tmid": [0.4, 0.6], "inc": [80.0, 90.0]}, + prior={"per": 2.0, "tmid": 0.5}, + duration_expected=5.0, + duration_measured=5.0, + transit_qc_expected_tmid=0.5, + transit_qc_expected_tmid_unc=0.001, + transit_qc_expected_rprs=0.10, + transit_qc_expected_rprs_unc=0.01, + transit_qc_use_deviation_from_expected_transit_in_qc=True, + transit_qc_deviation_sigma_threshold=5.0, + ) + + summary = evaluate_transit_detection_qc(fit) + + assert summary["status"] == "pass" + assert np.isnan(summary["tmid_deviation_minutes"]) + assert np.isnan(summary["tmid_deviation_sigma"]) + assert summary["rprs_deviation_sigma"] == pytest.approx(0.0) + assert summary["deviation_from_expected_value"] == pytest.approx(1.0) + assert not any("Expected-value Tmid" in note for note in summary["notes"]) + assert "QC rejected the fit because" not in summary["summary"] + assert "Tmid of the fit is 40.32 minutes away from the ephemeris Tmid" not in summary["summary"] + assert "not supported strongly enough against a flat/null model" not in summary["summary"] + + +def test_expected_value_rprs_deviation_uses_combined_uncertainty_with_systematic_floor(): + transit_model = np.ones(21, dtype=float) + transit_model[9:12] -= 0.0287 + data = transit_model.copy() + fit = types.SimpleNamespace( + data=data, + dataerr=np.full(data.shape[0], 0.0015, dtype=float), + model=transit_model, + airmass=np.ones(data.shape[0], dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.1694, "tmid": 0.5, "inc": 89.0, "a2": 0.0}, + errors={"rprs": 0.0046, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 0.5], "tmid": [0.4, 0.6], "inc": [80.0, 90.0]}, + duration_expected=5.0, + duration_measured=5.0, + transit_qc_expected_tmid=0.5, + transit_qc_expected_tmid_unc=0.001, + transit_qc_expected_rprs=0.1589, + transit_qc_expected_rprs_unc=0.0001, + transit_qc_use_deviation_from_expected_transit_in_qc=True, + transit_qc_deviation_sigma_threshold=5.0, + ) + + summary = evaluate_transit_detection_qc(fit) + + expected_comparison_unc = np.sqrt(0.0046 ** 2 + 0.0001 ** 2 + (0.05 * 0.1589) ** 2) + assert summary["rprs_deviation_fit_unc"] == pytest.approx(0.0046) + assert summary["rprs_deviation_expected_unc"] == pytest.approx(0.0001) + assert summary["rprs_deviation_systematic_floor"] == pytest.approx(0.05 * 0.1589) + assert summary["rprs_deviation_unc"] == pytest.approx(expected_comparison_unc) + assert summary["rprs_deviation_sigma"] == pytest.approx(abs(0.1694 - 0.1589) / expected_comparison_unc) + assert summary["deviation_from_expected_value"] == pytest.approx( + 1.0 - summary["rprs_deviation_sigma"] / 5.0 + ) + assert summary["deviation_from_expected_value"] > 0.0 + + +def test_expected_value_rprs_deviation_uses_model_data_fit_uncertainty(): + transit_model = np.ones(31, dtype=float) + transit_model[12:19] -= 0.0287 + residual_pattern = np.array( + [ + 0.0, 0.006, -0.005, 0.004, -0.006, 0.005, -0.004, 0.006, + -0.005, 0.004, -0.006, 0.005, -0.004, 0.006, -0.005, 0.004, + -0.006, 0.005, -0.004, 0.006, -0.005, 0.004, -0.006, 0.005, + -0.004, 0.006, -0.005, 0.004, -0.006, 0.005, 0.0, + ], + dtype=float, + ) + fit = types.SimpleNamespace( + time=np.linspace(0.0, 1.0, transit_model.size), + data=transit_model + residual_pattern, + dataerr=np.full(transit_model.size, 0.003, dtype=float), + model=transit_model, + transit=transit_model, + airmass=np.ones(transit_model.size, dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.1694, "tmid": 0.5, "inc": 89.0, "a2": 0.0}, + errors={"rprs": 0.0046, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 0.5], "tmid": [0.4, 0.6], "inc": [80.0, 90.0]}, + duration_expected=5.0, + duration_measured=5.0, + transit_qc_expected_tmid=0.5, + transit_qc_expected_tmid_unc=0.001, + transit_qc_expected_rprs=0.1589, + transit_qc_expected_rprs_unc=0.0001, + transit_qc_use_deviation_from_expected_transit_in_qc=True, + transit_qc_deviation_sigma_threshold=5.0, + ) + + summary = evaluate_transit_detection_qc(fit) + + expected_fit_unc = np.sqrt( + summary["rprs_deviation_model_fit_unc"] ** 2 + + summary["rprs_deviation_data_fit_unc"] ** 2 + ) + expected_comparison_unc = np.sqrt( + expected_fit_unc ** 2 + + summary["rprs_deviation_expected_unc"] ** 2 + + summary["rprs_deviation_systematic_floor"] ** 2 + ) + assert summary["rprs_deviation_model_fit_unc"] == pytest.approx(0.0046) + assert summary["rprs_deviation_data_fit_unc"] > 0.0 + assert summary["rprs_deviation_fit_unc"] == pytest.approx(expected_fit_unc) + assert summary["rprs_deviation_unc"] == pytest.approx(expected_comparison_unc) + contribution = next( + item for item in summary["ktmf_contributions"] + if item["label"] == "Deviation From Expected Value" + ) + assert "model uncertainty=0.004600" in contribution["detail"] + assert "data/red-noise uncertainty=" in contribution["detail"] + + +def test_selected_full_resolution_refit_keeps_expected_value_context(monkeypatch): + import exotic.exotic as exotic_module + + times = np.linspace(0.0, 1.0, 21) + transit_model = np.ones(times.shape[0], dtype=float) + transit_model[9:12] -= 0.0287 + errors = np.full(times.shape[0], 0.0015, dtype=float) + airmass = np.ones(times.shape[0], dtype=float) + + previous_fit = types.SimpleNamespace( + fast_ultranest_binning_applied=True, + parameters={ + "rprs": 0.1694, + "tmid": 0.5, + "ars": 5.0, + "inc": 89.0, + "per": 1.0, + "u0": 0.1, + "u1": 0.1, + "u2": 0.1, + "u3": 0.1, + "ecc": 0.0, + "omega": 0.0, + "a0": 1.0, + "a1": 1.0, + "a2": 0.0, + }, + errors={"rprs": 0.0046, "tmid": 0.001, "ars": 0.1, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + bounds={"rprs": [0.0, 0.5], "tmid": [0.4, 0.6], "ars": [1.0, 10.0], "inc": [80.0, 90.0]}, + ) + + def fake_run_nested(*args, **kwargs): + return types.SimpleNamespace( + time=times, + data=transit_model.copy(), + dataerr=errors.copy(), + model=transit_model.copy(), + airmass=airmass.copy(), + prior={"per": 1.0, "tmid": 0.5}, + parameters={ + "rprs": 0.1694, + "tmid": 0.5, + "ars": 5.0, + "inc": 89.0, + "per": 1.0, + "a0": 1.0, + "a1": 1.0, + "a2": 0.0, + }, + errors={"rprs": 0.0046, "tmid": 0.001, "ars": 0.1, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + bounds={"rprs": [0.0, 0.5], "tmid": [0.4, 0.6], "ars": [1.0, 10.0], "inc": [80.0, 90.0]}, + airmass_fit_skipped=True, + eebls_diagnostic_depth_snr=50.0, + duration_expected=0.1, + duration_measured=0.1, + ) + + monkeypatch.setattr(exotic_module, "run_nested_lightcurve_fit_with_rprs_posterior_retry", fake_run_nested) + monkeypatch.setattr(exotic_module, "build_expected_transit_coverage_assessment", lambda *args, **kwargs: {}) + monkeypatch.setattr(exotic_module, "log_expected_transit_coverage_assessment", lambda *args, **kwargs: None) + monkeypatch.setattr(exotic_module, "annotate_pre_ultranest_transit_coverage", lambda *args, **kwargs: None) + monkeypatch.setattr(exotic_module, "selected_final_live_point_target", lambda *args, **kwargs: (200, None)) + + p_dict = { + "rprs": 0.1589, + "rprsUnc": 0.0001, + "midT": 0.5, + "midTUnc": 0.001, + "pPer": 1.0, + "pPerUnc": 0.0, + "aRs": 5.0, + "aRsUnc": 0.1, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "use_deviation_from_expected_transit_in_qc": True, + "deviation_from_expected_transit_in_qc_sigma": 5.0, + } + selected_result = { + "fit": previous_fit, + "good_times": times, + "good_flux": transit_model.copy(), + "good_unc": errors.copy(), + "good_airmass": airmass.copy(), + "good_jd_times": times.copy(), + "fast_fit_bounds": previous_fit.bounds, + } + + refit, _, _ = refit_selected_fast_comparison_on_full_lightcurve( + selected_result, + p_dict, + detrend_on_outoftransit_baseline=False, + duration_prior={"duration": 0.1}, + ) + + expected_comparison_unc = np.sqrt(0.0046 ** 2 + 0.0001 ** 2 + (0.05 * 0.1589) ** 2) + assert refit.transit_qc_rprs_deviation_fit_unc == pytest.approx(0.0046) + assert refit.transit_qc_rprs_deviation_expected_unc == pytest.approx(0.0001) + assert refit.transit_qc_rprs_deviation_systematic_floor == pytest.approx(0.05 * 0.1589) + assert refit.transit_qc_rprs_deviation_unc == pytest.approx(expected_comparison_unc) + assert refit.transit_qc_rprs_deviation_sigma == pytest.approx(abs(0.1694 - 0.1589) / expected_comparison_unc) + contribution = next( + item for item in refit.transit_qc_ktmf_contributions + if item["label"] == "Deviation From Expected Value" + ) + assert contribution["score"] > 0.0 + assert contribution["max_points"] > 0.0 + assert "fit uncertainty=0.004600" in contribution["detail"] + assert "expected uncertainty=0.000100" in contribution["detail"] + assert "comparison uncertainty=" in contribution["detail"] + assert "systematic floor=" in contribution["detail"] + assert "Tmid" not in contribution["detail"] + + +def test_evaluate_transit_detection_qc_computes_missing_eebls_depth_snr(monkeypatch): + def fake_eebls(times, flux_values, flux_errors, prior, fallback_bounds): + return { + "method": "eebls", + "applied": True, + "tmid": 0.5, + "bounds": [0.45, 0.55], + "duration": 0.1, + "depth": 0.01, + "depth_snr": 7.25, + "note": "test eebls diagnostic", + } + + monkeypatch.setattr("exotic.exotic.estimate_tmid_and_bounds_with_eebls", fake_eebls) + + times = np.linspace(0.0, 1.0, 21) + transit_model = np.ones(times.shape[0], dtype=float) + transit_model[9:12] -= 0.02 + data = transit_model.copy() + fit = types.SimpleNamespace( + time=times, + data=data, + dataerr=np.full(data.shape[0], 0.0015, dtype=float), + model=transit_model, + airmass=np.ones(data.shape[0], dtype=float), + airmass_fit_skipped=True, + parameters={"rprs": 0.10, "tmid": 0.5, "inc": 89.0, "a2": 0.0, "per": 2.0}, + errors={"rprs": 0.01, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds={"rprs": [0.0, 1.0], "tmid": [0.4, 0.6], "inc": [80.0, 90.0]}, + prior={"per": 2.0, "tmid": 0.5}, + duration_expected=5.0, + duration_measured=5.0, + transit_qc_expected_tmid=0.5, + transit_qc_expected_tmid_unc=0.01, + transit_qc_expected_rprs=0.10, + transit_qc_expected_rprs_unc=0.05, + transit_qc_use_deviation_from_expected_transit_in_qc=False, + transit_qc_deviation_sigma_threshold=5.0, + ) + + summary = evaluate_transit_detection_qc(fit) + + assert summary["eebls_depth_snr"] == pytest.approx(7.25) + assert fit.eebls_diagnostic_depth_snr == pytest.approx(7.25) + + +def test_fit_final_lightcurve_with_oot_baseline_detrending_preserves_expected_tmid_context(monkeypatch): + run_count = {"value": 0, "duration_priors": []} + + def fake_run_nested(times, flux_values, flux_errors, airmass, prior, bounds, **kwargs): + run_count["value"] += 1 + run_count["duration_priors"].append(kwargs.get("duration_prior")) + local_times = np.asarray(times, dtype=float) + model = np.ones(local_times.shape[0], dtype=float) + model[1:-1] -= 0.01 + return types.SimpleNamespace( + time=local_times, + data=model.copy(), + dataerr=np.full(local_times.shape[0], 0.001, dtype=float), + model=model.copy(), + residuals=np.zeros(local_times.shape[0], dtype=float), + airmass=np.asarray(airmass, dtype=float), + prior=dict(prior), + parameters={"rprs": 0.1, "tmid": prior["tmid"], "inc": 89.0, "a2": 0.0, "per": prior["per"]}, + errors={"rprs": 0.01, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds=dict(bounds), + duration_expected=0.1, + duration_measured=0.1, + ) + + monkeypatch.setattr( + "exotic.exotic.run_nested_lightcurve_fit_with_rprs_posterior_retry", + fake_run_nested, + ) + monkeypatch.setattr("exotic.exotic.apply_plot_time_range", lambda fit, plot_time_range: fit) + monkeypatch.setattr("exotic.exotic.apply_vertical_flux_normalization_bound", lambda *args, **kwargs: None) + monkeypatch.setattr( + "exotic.exotic.build_final_fit_prefit_refinement_plan", + lambda times, flux_values, flux_errors, airmass, prior, bounds, fit, **kwargs: { + "applied": True, + "note": "test prefit refinement", + "times": np.asarray(times, dtype=float), + "flux": np.asarray(flux_values, dtype=float), + "unc": np.asarray(flux_errors, dtype=float), + "airmass": np.asarray(airmass, dtype=float), + "jd_times": None, + "prior": dict(prior), + "bounds": dict(bounds), + "duration": 0.1, + "original_point_count": len(times), + "refined_point_count": len(times), + "trimmed_pre_points": 0, + "trimmed_post_points": 0, + "original_tmid_bounds": bounds["tmid"], + "refined_tmid_bounds": bounds["tmid"], + }, + ) + + times = np.linspace(2460000.45, 2460000.55, 8) + fit, _, _ = fit_final_lightcurve_with_oot_baseline_detrending( + times, + np.ones(times.shape[0], dtype=float), + np.full(times.shape[0], 0.001, dtype=float), + np.linspace(1.0, 1.1, times.shape[0]), + {"rprs": 0.1, "tmid": 2460000.5, "inc": 89.0, "a2": 0.0, "per": 2.0}, + {"rprs": [0.0, 1.0], "tmid": [2460000.45, 2460000.55], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]}, + detrend_on_outoftransit_baseline=False, + expected_planet_dict={ + "midT": 2455000.0, + "midTUnc": 0.0001, + "pPer": 2.0, + "pPerUnc": 0.001, + "rprs": 0.1, + "rprsUnc": 0.01, + "aRs": 15.0, + "aRsUnc": 0.1, + "inc": 89.0, + "incUnc": 0.1, + "ecc": 0.0, + "omega": 0.0, + }, + expected_tmid_search_summary={ + "method": "ephemeris", + "applied": True, + "tmid": 2460000.5, + "uncertainty": 0.002, + "bounds": [2460000.45, 2460000.55], + "duration": 0.1, + "depth": np.nan, + "depth_snr": np.nan, + "note": "test propagated tmid", + }, + eebls_search_summary={ + "method": "eebls", + "applied": True, + "tmid": 2460000.5, + "bounds": [2460000.47, 2460000.53], + "duration": 0.1, + "depth": 0.01, + "depth_snr": 6.5, + "note": "test eebls", + }, + ) + + assert run_count["value"] == 2 + assert all(prior is not None and prior.get("applied") for prior in run_count["duration_priors"]) + assert fit.initial_tmid_search_tmid == pytest.approx(2460000.5) + assert fit.transit_qc_expected_tmid == pytest.approx(2460000.5) + assert fit.transit_qc_expected_tmid_unc == pytest.approx(0.002) + assert fit.eebls_diagnostic_depth_snr == pytest.approx(6.5) + + +def test_fit_ranked_comparison_calibration_candidates_selects_highest_ktmf_success(monkeypatch): + def fake_diagnostics(*args, **kwargs): + return {"usable_point_count": 6} + + def fake_finalize( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times=None, + **kwargs, + ): + comp_marker = int(np.nanmedian(cflux)) + residual_scale_map = {50: 0.05, 40: 0.02, 30: 0.03} + delta_bic_map = {50: 8.0, 40: 18.0, 30: 12.0} + ktmf_map = {50: 2.40, 40: 4.70, 30: 3.90} + residual_scale = residual_scale_map[comp_marker] + residuals = residual_scale * np.array([-1.0, 1.0, -1.0, 1.0, -1.0, 1.0], dtype=float) + fit = types.SimpleNamespace( + residuals=residuals, + data=np.ones_like(residuals), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + transit_qc_delta_bic=delta_bic_map[comp_marker], + transit_qc_ktmf_metric=ktmf_map[comp_marker], + ) + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": np.asarray(cflux, dtype=float), + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } + + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.2, 6) + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = {"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0} + aper_data = { + "target": np.full((6, 1, 1), 100.0, dtype=float), + "comp1": np.full((6, 1, 1), 50.0, dtype=float), + "comp2": np.full((6, 1, 1), 40.0, dtype=float), + "comp3": np.full((6, 1, 1), 30.0, dtype=float), + } + comparison_calibration = { + "method": "aperture", + "a": 0, + "an": 0, + "comp_summaries": [ + {"label": "Comp 1", "position": (10.0, 10.0), "aggregate_score": 0.01, "coverage_count": 6, "coverage_total_frame_count": 6, "coverage_reference_count": 6.0, "coverage_min_required_count": 5, "coverage_rejected": False, "comp_index": 0}, + {"label": "Comp 2", "position": (20.0, 20.0), "aggregate_score": 0.02, "coverage_count": 6, "coverage_total_frame_count": 6, "coverage_reference_count": 6.0, "coverage_min_required_count": 5, "coverage_rejected": False, "comp_index": 1}, + {"label": "Comp 3", "position": (30.0, 30.0), "aggregate_score": 0.03, "coverage_count": 6, "coverage_total_frame_count": 6, "coverage_reference_count": 6.0, "coverage_min_required_count": 5, "coverage_rejected": False, "comp_index": 2}, + ], + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld, + p_dict, + comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.full(6, 100.0, dtype=float), + ) + + assert len(result["attempts"]) == 3 + assert result["selection_metric"] == "ktmf_combined_quality" + assert result["selected_result"]["comp_index"] == 1 + assert result["selected_result"]["rank"] == 1 + assert result["selected_result"]["selected"] is True + assert result["selected_result"]["ktmf_metric"] == pytest.approx(4.70) + assert "highest KTMF/projected-scatter" in result["selected_result"]["selection_reason"] + assert result["attempts"][0]["selection_reason"].startswith( + "not selected: full-resolution UltraNest model residual scatter" + ) + + +def test_select_preferred_comparison_attempt_rejects_noisy_high_ktmf_before_ranking(): + attempts = [ + { + "label": "Comp 1", + "rank": 0, + "ktmf_metric": 4.8, + "residual_scatter": 0.040, + "eebls_snr": 3.0, + "transit_delta_bic": 10.0, + }, + { + "label": "Comp 2", + "rank": 1, + "ktmf_metric": 3.6, + "residual_scatter": 0.010, + "eebls_snr": 2.8, + "transit_delta_bic": 8.0, + }, + { + "label": "Comp 3", + "rank": 2, + "ktmf_metric": 3.8, + "residual_scatter": 0.014, + "eebls_snr": 2.6, + "transit_delta_bic": 7.0, + }, + ] + + selected, metric = select_preferred_comparison_attempt(attempts) + + assert metric == "ktmf_combined_quality" + assert selected["label"] == "Comp 2" + assert attempts[0]["scatter_gate_passed"] is False + assert attempts[0]["scatter_gate_threshold"] == pytest.approx(0.015) + assert selected["scatter_adjusted_ktmf_metric"] == pytest.approx(3.6) + assert attempts[2]["scatter_adjusted_ktmf_metric"] == pytest.approx(3.8 * 0.010 / 0.014) + + +def test_select_preferred_comparison_attempt_uses_ktmf_and_projected_selection_scatter_only(): + attempts = [ + { + "label": "Comp 1", + "rank": 0, + "ktmf_metric": 2.59, + "selection_scatter": 0.022659, + "target_comp_scatter": 0.005693, + "aggregate_score": 0.010, + "eebls_snr": 3.27, + "transit_delta_bic": 2.71, + }, + { + "label": "Comp 2", + "rank": 1, + "ktmf_metric": 3.23, + "selection_scatter": 0.027437, + "target_comp_scatter": 0.003000, + "aggregate_score": 0.001, + "eebls_snr": 3.53, + "transit_delta_bic": 2.28, + }, + { + "label": "Comp 9", + "rank": 2, + "ktmf_metric": 3.25, + "selection_scatter": 0.023904, + "target_comp_scatter": 0.030258, + "aggregate_score": 0.100, + "eebls_snr": 1.61, + "transit_delta_bic": -8.36, + }, + ] + + selected, metric = select_preferred_comparison_attempt(attempts) + + assert metric == "ktmf_combined_quality" + assert selected["label"] == "Comp 9" + assert attempts[0]["combined_quality_ktmf_metric"] == pytest.approx(2.59 / 2.2659) + assert attempts[1]["combined_quality_ktmf_metric"] == pytest.approx(3.23 / 2.7437) + assert attempts[2]["combined_quality_ktmf_metric"] == pytest.approx(3.25 / 2.3904) + assert selected["combined_quality_ktmf_metric"] > attempts[1]["combined_quality_ktmf_metric"] + assert selected["combined_quality_ktmf_metric"] > attempts[0]["combined_quality_ktmf_metric"] + + +def test_target_comp_flux_scatter_measures_normalized_target_reference_ratio(): + comp_flux = np.full(8, 100.0, dtype=float) + ratio = np.array([1.00, 1.01, 0.99, 1.02, 0.98, 1.00, 1.01, 0.99], dtype=float) + target_flux = comp_flux * ratio + + scatter = target_comp_flux_scatter(target_flux, comp_flux, min_points=5) + + assert scatter == pytest.approx(0.014826, rel=1.0e-3) + + +def test_fitted_lightcurve_scatter_on_dataset_projects_fit_to_full_flux(monkeypatch): + def fake_transit(times, parameters): + return np.ones_like(np.asarray(times, dtype=float)) + + monkeypatch.setattr("exotic.exotic.transit", fake_transit) + fit = types.SimpleNamespace( + parameters={"a0": 1.0, "a2": 0.0}, + airmass_reference=1.0, + ) + times = np.arange(8, dtype=float) + flux_values = np.array([1.0, 1.01, 0.99, 1.02, 0.98, 1.0, 1.01, 0.99], dtype=float) + airmass = np.ones_like(times) + + scatter = fitted_lightcurve_scatter_on_dataset(fit, times, flux_values, airmass) + + assert scatter == pytest.approx(np.std(flux_values - 1.0) / np.median(flux_values)) + + +def test_fit_ranked_comparison_calibration_candidates_extends_only_selected_final_fit(monkeypatch): + monkeypatch.setenv("EXOTIC_ULTRANEST_MIN_NUM_LIVE_POINTS", "200") + monkeypatch.setenv("EXOTIC_SPARSE_POSTERIOR_LIVE_POINT_RETRY", "1") + + def fake_diagnostics(*args, **kwargs): + return {"usable_point_count": 6} + + created_fits = {} + + class RetainedSamplerFit: + def __init__(self, comp_marker, ktmf_metric): + self.comp_marker = comp_marker + self.extension_calls = [] + self.cleared = False + self.max_ncalls = 1000 + self.time = np.linspace(0.0, 0.05, 6) + self.data = np.ones(6, dtype=float) + self.residuals = np.full(6, 0.01, dtype=float) + self.parameters = { + "tmid": 0.5, + "rprs": 0.1, + "inc": 89.0, + "ars": 10.0, + "a0": 1.0, + "a2": 0.0, + } + self.errors = { + "tmid": 0.001, + "rprs": 0.001, + "inc": 0.1, + "ars": 0.1, + "a0": 0.01, + "a2": 0.01, + } + self.bounds = { + "rprs": [0.08, 0.12], + "tmid": [0.49, 0.51], + "ars": [9.0, 11.0], + "inc": [85.0, 90.0], + "a2": [-3.0, 3.0], + } + self.transit_qc_delta_bic = 12.0 + comp_marker / 100.0 + self.transit_qc_ktmf_metric = ktmf_metric + + def get_parameter_posterior_samples(self, key): + ranges = { + "rprs": (0.09, 0.11), + "tmid": (0.499, 0.501), + "ars": (9.5, 10.5), + } + low, high = ranges[key] + return np.linspace(low, high, 1500) + + def extend_ultranest_fit(self, min_num_live_points=None, max_ncalls=None): + self.extension_calls.append({ + "min_num_live_points": min_num_live_points, + "max_ncalls": max_ncalls, + "bounds": self.bounds.copy(), + }) + return True + + def clear_ultranest_resume_state(self): + self.cleared = True + + def fake_finalize( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times=None, + **kwargs, + ): + comp_marker = int(np.nanmedian(cflux)) + ktmf_map = {50: 2.40, 40: 4.70, 30: 3.90} + fit = RetainedSamplerFit(comp_marker, ktmf_map[comp_marker]) + created_fits[comp_marker] = fit + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": np.asarray(cflux, dtype=float), + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } + + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.2, 6) + aper_data = { + "target": np.full((6, 1, 1), 100.0, dtype=float), + "comp1": np.full((6, 1, 1), 50.0, dtype=float), + "comp2": np.full((6, 1, 1), 40.0, dtype=float), + "comp3": np.full((6, 1, 1), 30.0, dtype=float), + } + comparison_calibration = { + "method": "aperture", + "a": 0, + "an": 0, + "comp_summaries": [ + {"label": "Comp 1", "aggregate_score": 0.01, "coverage_rejected": False, "comp_index": 0}, + {"label": "Comp 2", "aggregate_score": 0.02, "coverage_rejected": False, "comp_index": 1}, + {"label": "Comp 3", "aggregate_score": 0.03, "coverage_rejected": False, "comp_index": 2}, + ], + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.full(6, 100.0, dtype=float), + ) + + assert result["selected_result"]["comp_index"] == 1 + assert created_fits[40].extension_calls == [{ + "min_num_live_points": 1200, + "max_ncalls": 6000, + "bounds": created_fits[40].bounds, + }] + assert created_fits[50].extension_calls == [] + assert created_fits[30].extension_calls == [] + assert created_fits[40].cleared is True + assert created_fits[50].cleared is True + assert created_fits[30].cleared is True + assert "selected comparison-star final" in created_fits[40].sparse_posterior_live_point_extension_note + + +def test_fit_ranked_comparison_calibration_candidates_stops_at_first_qc_pass_by_default(monkeypatch): + def fake_diagnostics(*args, **kwargs): + return {"usable_point_count": 6} + + call_markers = [] + + def fake_finalize( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times=None, + **kwargs, + ): + comp_marker = int(np.nanmedian(cflux)) + call_markers.append(comp_marker) + status_map = {50: "marginal", 40: "pass", 30: "pass"} + ktmf_map = {50: 4.90, 40: 3.20, 30: 5.00} + status = status_map[comp_marker] + ktmf_metric = ktmf_map[comp_marker] + fit = types.SimpleNamespace( + residuals=np.full(6, 0.01, dtype=float), + data=np.ones(6, dtype=float), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + transit_qc={ + "status": status, + "summary": "ok", + "delta_bic": 12.0 + comp_marker / 100.0, + "ktmf_metric": ktmf_metric, + "ktmf_contributions": [], + }, + transit_qc_status=status, + transit_qc_ktmf_metric=ktmf_metric, + transit_qc_delta_bic=12.0 + comp_marker / 100.0, + ) + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": np.asarray(cflux, dtype=float), + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } + + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.2, 6) + aper_data = { + "target": np.full((6, 1, 1), 100.0, dtype=float), + "comp1": np.full((6, 1, 1), 50.0, dtype=float), + "comp2": np.full((6, 1, 1), 40.0, dtype=float), + "comp3": np.full((6, 1, 1), 30.0, dtype=float), + } + comparison_calibration = { + "method": "aperture", + "a": 0, + "an": 0, + "comp_summaries": [ + {"label": "Comp 1", "aggregate_score": 0.01, "coverage_rejected": False, "comp_index": 0}, + {"label": "Comp 2", "aggregate_score": 0.02, "coverage_rejected": False, "comp_index": 1}, + {"label": "Comp 3", "aggregate_score": 0.03, "coverage_rejected": False, "comp_index": 2}, + ], + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.full(6, 100.0, dtype=float), + ) + + assert call_markers == [50, 40] + assert len(result["attempts"]) == 2 + assert result["stopped_after_first_qc_pass"] is True + assert result["selection_metric"] == "first_qc_pass" + assert result["selected_result"]["comp_index"] == 1 + assert result["selected_result"]["search_stopped_after_qc_pass"] is True + assert "first completed comparison-star candidate" in result["selected_result"]["selection_reason"] + + +def test_fit_ranked_comparison_calibration_candidates_stops_at_promising_partial_marginal(monkeypatch): + def fake_diagnostics(*args, **kwargs): + return {"usable_point_count": 6} + + monkeypatch.setattr( + "exotic.exotic.build_comparison_candidate_preflight", + lambda *args, **kwargs: { + "prepared_series": None, + "coverage_priority": 2, + "scout": {"score": np.nan}, + }, + ) + + call_markers = [] + + def fake_finalize( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times=None, + **kwargs, + ): + comp_marker = int(np.nanmedian(cflux)) + call_markers.append(comp_marker) + status = "marginal" if comp_marker == 50 else "pass" + fit = types.SimpleNamespace( + residuals=np.full(6, 0.01, dtype=float), + data=np.ones(6, dtype=float), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + transit_qc={"status": status, "summary": "ok"}, + transit_qc_status=status, + transit_qc_ktmf_metric=3.5, + transit_qc_delta_bic=15.1, + ) + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": np.asarray(cflux, dtype=float), + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } + + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.2, 6) + aper_data = { + "target": np.full((6, 1, 1), 100.0, dtype=float), + "comp1": np.full((6, 1, 1), 50.0, dtype=float), + "comp2": np.full((6, 1, 1), 40.0, dtype=float), + } + comparison_calibration = { + "method": "aperture", + "a": 0, + "an": 0, + "comp_summaries": [ + {"label": "Comp 1", "aggregate_score": 0.01, "coverage_rejected": False, "comp_index": 0}, + {"label": "Comp 2", "aggregate_score": 0.02, "coverage_rejected": False, "comp_index": 1}, + ], + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.full(6, 100.0, dtype=float), + ) + + assert call_markers == [50] + assert result["selection_metric"] == "promising_partial" + assert result["selected_result"]["search_stopped_after_promising_partial"] is True + assert result["stopped_after_promising_partial"] is True + + +def test_fit_ranked_comparison_calibration_candidates_can_evaluate_all_qc_passes_when_exit_disabled(monkeypatch): + def fake_diagnostics(*args, **kwargs): + return {"usable_point_count": 6} + + call_markers = [] + + def fake_finalize( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times=None, + **kwargs, + ): + comp_marker = int(np.nanmedian(cflux)) + call_markers.append(comp_marker) + ktmf_metric = {50: 3.10, 40: 4.00, 30: 4.80}[comp_marker] + fit = types.SimpleNamespace( + residuals=np.full(6, 0.01, dtype=float), + data=np.ones(6, dtype=float), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + transit_qc={ + "status": "pass", + "summary": "ok", + "delta_bic": 12.0 + comp_marker / 100.0, + "ktmf_metric": ktmf_metric, + "ktmf_contributions": [], + }, + transit_qc_status="pass", + transit_qc_ktmf_metric=ktmf_metric, + transit_qc_delta_bic=12.0 + comp_marker / 100.0, + ) + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": np.asarray(cflux, dtype=float), + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } + + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.2, 6) + aper_data = { + "target": np.full((6, 1, 1), 100.0, dtype=float), + "comp1": np.full((6, 1, 1), 50.0, dtype=float), + "comp2": np.full((6, 1, 1), 40.0, dtype=float), + "comp3": np.full((6, 1, 1), 30.0, dtype=float), + } + comparison_calibration = { + "method": "aperture", + "a": 0, + "an": 0, + "comp_summaries": [ + {"label": "Comp 1", "aggregate_score": 0.01, "coverage_rejected": False, "comp_index": 0}, + {"label": "Comp 2", "aggregate_score": 0.02, "coverage_rejected": False, "comp_index": 1}, + {"label": "Comp 3", "aggregate_score": 0.03, "coverage_rejected": False, "comp_index": 2}, + ], + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.full(6, 100.0, dtype=float), + exit_at_first_qc_pass_solution=False, + ) + + assert call_markers == [50, 40, 30] + assert result["stopped_after_first_qc_pass"] is False + assert result["selection_metric"] == "ktmf_combined_quality" + assert result["selected_result"]["comp_index"] == 2 + assert result["selected_result"]["ktmf_metric"] == pytest.approx(4.80) + + +def test_ranked_comparison_calibration_summaries_skip_suitability_outliers(): + ranked = ranked_comparison_calibration_summaries( + { + "comp_summaries": [ + {"comp_index": 0, "aggregate_score": 0.139668, "coverage_rejected": False, "suitability_outlier_rejected": True}, + {"comp_index": 1, "aggregate_score": 0.051809, "coverage_rejected": False, "suitability_outlier_rejected": True}, + {"comp_index": 2, "aggregate_score": 0.024802, "coverage_rejected": False, "suitability_outlier_rejected": False}, + {"comp_index": 3, "aggregate_score": 0.027680, "coverage_rejected": False, "suitability_outlier_rejected": False}, + ] + } + ) + + assert [summary["comp_index"] for summary in ranked] == [2, 3] + + +def test_comparison_preflight_ranking_prioritizes_full_coverage_then_scout_score(): + plans = [ + { + "field_rank": 0, + "summary": {"comp_index": 7, "aggregate_score": 0.002175, "label": "Comp 8"}, + "preflight": {"coverage_priority": 2, "scout": {"score": 0.42}}, + }, + { + "field_rank": 1, + "summary": {"comp_index": 0, "aggregate_score": 0.002331, "label": "Comp 1"}, + "preflight": {"coverage_priority": 2, "scout": {"score": 0.91}}, + }, + { + "field_rank": 4, + "summary": {"comp_index": 2, "aggregate_score": 0.002804, "label": "Comp 3"}, + "preflight": {"coverage_priority": 0, "scout": {"score": 0.25}}, + }, + ] + + ranked = rank_comparison_candidate_preflight_plans(plans) + + assert [plan["summary"]["comp_index"] for plan in ranked] == [2, 0, 7] + + +def test_promising_partial_comparison_attempt_can_stop_candidate_search(): + attempt = { + "fit": object(), + "full_reduction_applied": True, + "rejected_by_transit_qc": False, + "transit_qc_status": "marginal", + "preflight_coverage_priority": 2, + "ktmf_metric": 3.50, + "transit_delta_bic": 15.09, + } + + assert should_stop_after_promising_partial_comparison_attempt(attempt) is True + + attempt["preflight_coverage_priority"] = 4 + assert should_stop_after_promising_partial_comparison_attempt(attempt) is False + + +def test_fit_ranked_comparison_calibration_candidates_applies_field_image_clip(monkeypatch): + observed_lengths = [] + + def fake_diagnostics(times, *args, **kwargs): + observed_lengths.append(len(times)) + return {"usable_point_count": len(times)} + + def fake_finalize( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times=None, + **kwargs, + ): + observed_lengths.append(len(times)) + fit = types.SimpleNamespace( + residuals=np.full(len(times), 0.01, dtype=float), + data=np.ones(len(times), dtype=float), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + transit_qc={"status": "pass", "summary": "ok", "ktmf_metric": 4.2}, + transit_qc_status="pass", + transit_qc_summary="ok", + transit_qc_ktmf_metric=4.2, + transit_qc_delta_bic=16.0, + frame_filter_diagnostics=[{"stage": "Comparison-field image clip", "dropped_point_count": 2}], + ) + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": np.asarray(cflux, dtype=float), + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } + + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.2, 6) + comparison_calibration = { + "method": "aperture", + "method_label": "Aperture photometry (aper=5.00px, annulus=12.00px)", + "a": 0, + "an": 0, + "aper": 5.0, + "annulus": 12.0, + "field_image_keep_mask": np.array([True, False, True, True, False, True], dtype=bool), + "image_outlier_sigma": 4.25, + "image_outlier_required_valid_pairs": 2, + "comp_summaries": [ + {"label": "Comp 1", "position": (10.0, 10.0), "aggregate_score": 0.01, "coverage_count": 6, "coverage_total_frame_count": 6, "coverage_reference_count": 6.0, "coverage_min_required_count": 5, "coverage_rejected": False, "comp_index": 0}, + ], + } + aper_data = { + "target": np.full((6, 1, 1), 100.0, dtype=float), + "comp1": np.full((6, 1, 1), 50.0, dtype=float), + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.full(6, 100.0, dtype=float), + ) + + assert observed_lengths == [4, 4] + diagnostic = result["attempts"][0]["fit"].frame_filter_diagnostics[0] + assert diagnostic["stage"] == "Comparison-field image clip" + assert diagnostic["dropped_point_count"] == 2 + assert result["attempts"][0]["fit_point_count"] == 4 + + +def test_fit_ranked_comparison_calibration_candidates_masks_target_psf_shape(monkeypatch): + observed_lengths = [] + + def fake_diagnostics(times, *args, **kwargs): + observed_lengths.append(("diagnostics", len(times))) + return {"usable_point_count": len(times), "failure_reason": None} + + def fake_preflight(*args, **kwargs): + return {"coverage_priority": 1, "prepared_series": None} + + def fake_finalize( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times=None, + **kwargs, + ): + observed_lengths.append(("finalize", len(times))) + fit = types.SimpleNamespace( + residuals=np.full(len(times), 0.01, dtype=float), + data=np.ones(len(times), dtype=float), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + transit_qc={"status": "pass", "summary": "ok", "ktmf_metric": 4.2}, + transit_qc_status="pass", + transit_qc_summary="ok", + transit_qc_ktmf_metric=4.2, + transit_qc_delta_bic=16.0, + frame_filter_diagnostics=[], + ) + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": np.asarray(cflux, dtype=float), + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } + + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.build_comparison_candidate_preflight", fake_preflight) + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + frame_count = 30 + times = np.linspace(0.0, 0.2, frame_count) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.2, frame_count) + + psf_rows = np.zeros((frame_count, 7), dtype=float) + psf_rows[:, 0] = 10.0 + psf_rows[:, 1] = 20.0 + psf_rows[:, 2] = 100.0 + psf_rows[:, 3] = 1.0 + psf_rows[:, 4] = 1.0 + psf_data = { + "target": psf_rows.copy(), + "comp1": psf_rows.copy(), + } + psf_data["comp1"][:, 2] = 120.0 + psf_data["target"][12, 3:5] = 6.5 + psf_flux_data = { + "target": psf_data["target"].copy(), + "comp1": psf_data["comp1"].copy(), + } + psf_flux_data["comp1"][:, 2] = 240.0 + + target_psf_flux = 2 * np.pi * psf_data["target"][:, 2] * psf_data["target"][:, 3] * psf_data["target"][:, 4] + comparison_calibration = { + "method": "psf", + "method_label": "PSF photometry", + "a": None, + "an": None, + "aper": 0.0, + "annulus": 15.0, + "comp_summaries": [ + { + "label": "Comp 1", + "position": (10.0, 10.0), + "aggregate_score": 0.01, + "coverage_count": frame_count, + "coverage_total_frame_count": frame_count, + "coverage_reference_count": float(frame_count), + "coverage_min_required_count": 5, + "coverage_rejected": False, + "comp_index": 0, + "key": "comp1", + }, + ], + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0}, + comparison_calibration=comparison_calibration, + psf_data=psf_data, + aper_data=None, + target_psf_flux=target_psf_flux, + psf_flux_data=psf_flux_data, + ) + + assert observed_lengths == [("diagnostics", frame_count - 1), ("finalize", frame_count - 1)] + assert result["selected_result"]["fit_point_count"] == frame_count - 1 + assert np.nanmax(result["selected_result"]["tflux_fit"]) < 1000.0 + assert np.nanmedian(result["selected_result"]["cflux_fit"]) == pytest.approx(2.0 * np.pi * 240.0) + + +def test_fit_ranked_comparison_calibration_candidates_applies_candidate_intercomparison_clip(monkeypatch): + observed_lengths = [] + + def fake_diagnostics(times, *args, **kwargs): + observed_lengths.append(("diagnostics", len(times))) + return {"usable_point_count": len(times), "failure_reason": None} + + def fake_preflight(*args, **kwargs): + return {"coverage_priority": 1, "prepared_series": None} + + def fake_finalize( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times=None, + **kwargs, + ): + observed_lengths.append(("finalize", len(times))) + fit = types.SimpleNamespace( + residuals=np.full(len(times), 0.01, dtype=float), + data=np.ones(len(times), dtype=float), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + transit_qc={"status": "pass", "summary": "ok", "ktmf_metric": 4.2}, + transit_qc_status="pass", + transit_qc_summary="ok", + transit_qc_ktmf_metric=4.2, + transit_qc_delta_bic=16.0, + frame_filter_diagnostics=[], + ) + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": np.asarray(cflux, dtype=float), + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } + + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.build_comparison_candidate_preflight", fake_preflight) + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.2, 6) + comparison_calibration = { + "method": "aperture", + "method_label": "Aperture photometry (aper=5.00px, annulus=12.00px)", + "a": 0, + "an": 0, + "aper": 5.0, + "annulus": 12.0, + "field_image_keep_mask": np.ones(6, dtype=bool), + "comp_summaries": [ + { + "label": "Comp 1", + "position": (10.0, 10.0), + "aggregate_score": 0.01, + "coverage_count": 6, + "coverage_total_frame_count": 6, + "coverage_reference_count": 6.0, + "coverage_min_required_count": 5, + "coverage_rejected": False, + "suitability_outlier_rejected": False, + "comp_index": 0, + "ensemble_frame_keep_mask": np.array([True, True, False, True, True, True], dtype=bool), + "ensemble_frame_required_valid_pairs": 2, + "ensemble_frame_sigma": 4.25, + }, + ], + } + aper_data = { + "target": np.full((6, 1, 1), 100.0, dtype=float), + "comp1": np.full((6, 1, 1), 50.0, dtype=float), + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.full(6, 100.0, dtype=float), + ) + + assert observed_lengths == [("diagnostics", 5), ("finalize", 5)] + diagnostic = result["attempts"][0]["fit"].frame_filter_diagnostics[0] + assert diagnostic["stage"] == "Comparison-candidate intercomparison clip" + assert diagnostic["dropped_point_count"] == 1 + assert result["attempts"][0]["fit_point_count"] == 5 + + +def test_fit_ranked_comparison_calibration_candidates_saves_outputs_for_completed_candidates( + monkeypatch, tmp_path +): + def fake_diagnostics(*args, **kwargs): + return {"usable_point_count": 6} + + call_markers = [] + + def fake_finalize( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times=None, + **kwargs, + ): + comp_marker = int(np.nanmedian(cflux)) + call_markers.append(comp_marker) + fit = types.SimpleNamespace( + residuals=np.full(6, 0.01, dtype=float), + data=np.ones(6, dtype=float), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + transit_qc_ktmf_metric={50: 3.2, 40: 4.4}[comp_marker], + transit_qc_delta_bic=12.0 + comp_marker / 100.0, + ) + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": np.asarray(cflux, dtype=float), + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } + + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + saved_dirs = [] + + def fake_save(save_dir, provisional_fit, final_fit, p_dict, observation_date, comp_index, **kwargs): + candidate_dir = Path(save_dir) / f"comp{comp_index + 1}" + candidate_dir.mkdir(parents=True, exist_ok=True) + saved_dirs.append(candidate_dir) + return candidate_dir + + monkeypatch.setattr( + "exotic.exotic.save_comparison_candidate_full_reduction_outputs", + fake_save, + ) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.2, 6) + aper_data = { + "target": np.full((6, 1, 1), 100.0, dtype=float), + "comp1": np.full((6, 1, 1), 50.0, dtype=float), + "comp2": np.full((6, 1, 1), 40.0, dtype=float), + } + comparison_calibration = { + "method": "aperture", + "a": 0, + "an": 0, + "comp_summaries": [ + {"label": "Comp 1", "aggregate_score": 0.01, "coverage_rejected": False, "comp_index": 0}, + {"label": "Comp 2", "aggregate_score": 0.02, "coverage_rejected": False, "comp_index": 1}, + ], + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.full(6, 100.0, dtype=float), + save_dir=tmp_path, + planet_name="HAT-P-32 b", + observation_date="2026-04-28", + ) + + assert call_markers == [50, 40] + assert len(result["attempts"]) == 2 + assert result["selection_metric"] == "ktmf_combined_quality" + assert result["selected_result"]["comp_index"] == 1 + assert [attempt["final_output_dir"] for attempt in result["attempts"]] == [ + str(tmp_path / "comp1"), + str(tmp_path / "comp2"), + ] + assert saved_dirs == [tmp_path / "comp1", tmp_path / "comp2"] + + +def test_fit_ranked_comparison_calibration_candidates_can_prefer_highest_eebls_snr(monkeypatch): + def fake_diagnostics(*args, **kwargs): + return {"usable_point_count": 6} + + def fake_finalize( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times=None, + **kwargs, + ): + comp_marker = int(np.nanmedian(cflux)) + if comp_marker == 50: + residual_level = 0.01 + eebls_snr = 4.0 + else: + residual_level = 0.012 + eebls_snr = 7.5 + + residuals = residual_level * np.array([-1.0, 1.0, -1.0, 1.0, -1.0, 1.0], dtype=float) + fit = types.SimpleNamespace( + residuals=residuals, + data=np.ones_like(residuals), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + eebls_diagnostic_depth_snr=eebls_snr, + transit_qc_delta_bic=(10.0 if comp_marker == 50 else 12.0), + ) + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": np.asarray(cflux, dtype=float), + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } + + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.2, 6) + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = {"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0} + aper_data = { + "target": np.full((6, 1, 1), 100.0, dtype=float), + "comp1": np.full((6, 1, 1), 50.0, dtype=float), + "comp2": np.full((6, 1, 1), 40.0, dtype=float), + } + comparison_calibration = { + "method": "aperture", + "a": 0, + "an": 0, + "comp_summaries": [ + {"label": "Comp 1", "position": (10.0, 10.0), "aggregate_score": 0.01, "coverage_count": 6, "coverage_total_frame_count": 6, "coverage_reference_count": 6.0, "coverage_min_required_count": 5, "coverage_rejected": False, "comp_index": 0}, + {"label": "Comp 2", "position": (20.0, 20.0), "aggregate_score": 0.02, "coverage_count": 6, "coverage_total_frame_count": 6, "coverage_reference_count": 6.0, "coverage_min_required_count": 5, "coverage_rejected": False, "comp_index": 1}, + ], + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld, + p_dict, + comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.full(6, 100.0, dtype=float), + pick_comparison_by_eebls_snr=True, + ) + + assert result["selection_metric"] == "eebls_snr" + assert result["selected_result"]["comp_index"] == 1 + assert result["selected_result"]["eebls_snr"] == pytest.approx(7.5) + assert "highest selection-pass EEBLS SNR" in result["selected_result"]["selection_reason"] + assert result["attempts"][0]["selection_reason"].startswith("not selected: selection-pass EEBLS SNR") + + +def test_fit_ranked_comparison_calibration_candidates_logs_per_comp_run_reporting(monkeypatch): + logged = [] + + def fake_diagnostics(*args, **kwargs): + return {"usable_point_count": 6} + + final_fit = types.SimpleNamespace( + residuals=np.full(6, 0.01, dtype=float), + data=np.ones(6, dtype=float), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + ns_type="ultranest", + transit_qc={ + "status": "pass", + "summary": "Transit model strongly preferred over flat/null model.", + "delta_bic": 18.4, + "residual_scatter": 0.0035, + "ktmf_metric": 4.6, + "ktmf_contributions": [], + }, + transit_qc_status="pass", + transit_qc_summary="Transit model strongly preferred over flat/null model.", + transit_qc_delta_bic=18.4, + transit_qc_residual_scatter=0.0035, + transit_qc_ktmf_metric=4.6, + transit_qc_ktmf_contributions=[], + rprs_posterior_refit_applied=True, + rprs_posterior_refit_count=1, + rprs_posterior_refit_note="Applied 1 automatic Rp/R* posterior range refit(s).", + prefit_refinement_applied=True, + prefit_refinement_note="Applied a focused final-fit prefit refinement window.", + oot_baseline_detrending_applied=False, + oot_baseline_detrending_note="Skipped; need out-of-transit coverage on both sides of transit to fit a linear baseline.", + ) + + def fake_finalize(times, tflux, cflux, airmass, ld, p_dict, jd_times=None, **kwargs): + return { + "applied": True, + "fit": final_fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": np.asarray(cflux, dtype=float), + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "completed the full comparison-candidate reduction.", + } + + monkeypatch.setattr("exotic.exotic.log_info", lambda message, warn=False, error=False: logged.append(message)) + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.2, 6) + aper_data = { + "target": np.full((6, 1, 1), 100.0, dtype=float), + "comp1": np.full((6, 1, 1), 50.0, dtype=float), + } + comparison_calibration = { + "method": "aperture", + "method_label": "Aperture photometry (aper=5.00px, annulus=12.00px)", + "a": 0, + "an": 0, + "comp_summaries": [ + { + "label": "Comp 1", + "position": (10.0, 10.0), + "aggregate_score": 0.01, + "coverage_count": 6, + "coverage_total_frame_count": 6, + "coverage_reference_count": 6.0, + "coverage_min_required_count": 5, + "coverage_rejected": False, + "comp_index": 0, + }, + ], + } + + fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.full(6, 100.0, dtype=float), + ) + + assert any("Starting comparison-star target-fit evaluation for Comp 1" in message for message in logged) + assert any("Preparing comparison-candidate light curve for the full reduction." in message for message in logged) + assert any("Full reduction starting. Optional out-of-transit baseline detrending is enabled." in message for message in logged) + assert any("Completed comparison-star target-fit evaluation for Comp 1" in message and "transit_qc=PASS" in message for message in logged) + assert any("Rp/R* posterior retry note: Applied 1 automatic Rp/R* posterior range refit(s)." in message for message in logged) + assert any("OOT baseline detrending note: Skipped; need out-of-transit coverage on both sides of transit to fit a linear baseline." in message for message in logged) + + +def test_evaluate_lightcurve_candidate_requests_nested_fit(monkeypatch): + def fake_diagnostics(*args, **kwargs): + return {"usable_point_count": 6} + + def fake_fit_lightcurve( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times=None, + **kwargs, + ): + assert kwargs.get("final_fit_mode") == "ns" + fit = types.SimpleNamespace( + residuals=np.full(6, 0.01, dtype=float), + data=np.ones(6, dtype=float), + parameters={"tmid": 0.5, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0}, + errors={"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + ns_type="ultranest", + transit_qc={"status": "pass", "delta_bic": 12.0, "ktmf_metric": 3.8, "ktmf_contributions": []}, + transit_qc_status="pass", + transit_qc_summary="ok", + transit_qc_delta_bic=12.0, + transit_qc_ktmf_metric=3.8, + ) + return fit, np.asarray(tflux, dtype=float), np.asarray(cflux, dtype=float) + + monkeypatch.setattr("exotic.exotic.diagnose_lightcurve_fit_inputs", fake_diagnostics) + monkeypatch.setattr("exotic.exotic.fit_lightcurve", fake_fit_lightcurve) + + result, tflux_fit, cflux_fit = evaluate_lightcurve_candidate( + ( + np.linspace(0.0, 0.05, 6), + np.full(6, 20.0), + np.full(6, 10.0), + np.linspace(1.0, 1.2, 6), + [0.1, 0.1, 0.1, 0.1], + {"midT": 0.5, "pPer": 1.0, "rprs": 0.1, "aRs": 10.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0}, + 2460000.0 + np.linspace(0.0, 0.05, 6), + None, + False, + True, + True, + True, + ) + ) + + assert result["accepted"] is True + assert result["ktmf_metric"] == pytest.approx(3.8) + assert tflux_fit.shape == (6,) + assert cflux_fit.shape == (6,) + + +def test_fit_lightcurve_refines_nested_tmid_bounds_from_two_sided_lm_fit(monkeypatch): + captured_calls = [] + + def fake_lc_fitter( + times, + fluxes, + flux_unc, + airmass, + prior, + bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + captured_calls.append({ + "mode": mode, + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in bounds.items() + }, + }) + if mode == "lm": + transit = np.ones_like(times, dtype=float) + transit[(times >= 0.018) & (times <= 0.032)] = 0.98 + return types.SimpleNamespace( + transit=transit, + parameters={"tmid": 0.025, "rprs": 0.1, "inc": 89.0, "a2": 0.0, "per": 1.0}, + duration_expected=0.014, + ) + return types.SimpleNamespace(parameters={"tmid": 0.025, "rprs": 0.1, "inc": 89.0, "a2": 0.0}) + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 0.05, 21) + tflux = np.full(times.shape[0], 2.0) + cflux = np.full(times.shape[0], 2.0) + airmass = np.linspace(1.0, 1.5, times.shape[0]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + + myfit, _, _ = fit_lightcurve( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times, + final_fit_mode="ns", + ) + + assert myfit is not None + assert captured_calls[0]["mode"] == "lm" + assert captured_calls[1]["mode"] == "ns" + assert captured_calls[0]["bounds"]["tmid"] == pytest.approx([0.011347361950458953, 0.03865263804954105]) + assert captured_calls[1]["bounds"]["tmid"] == pytest.approx([0.0175, 0.0325]) + assert myfit.nested_tmid_refinement_applied is True + assert "recenter nested-sampling Tmid bounds" in myfit.nested_tmid_refinement_note + + +def test_fit_lightcurve_skips_nested_tmid_refinement_for_one_sided_lm_fit(monkeypatch): + captured_calls = [] + + def fake_lc_fitter( + times, + fluxes, + flux_unc, + airmass, + prior, + bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + captured_calls.append({ + "mode": mode, + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in bounds.items() + }, + }) + if mode == "lm": + transit = np.ones_like(times, dtype=float) + transit[times >= 0.025] = 0.98 + return types.SimpleNamespace( + transit=transit, + parameters={"tmid": 0.025, "rprs": 0.1, "inc": 89.0, "a2": 0.0, "per": 1.0}, + duration_expected=0.014, + ) + return types.SimpleNamespace(parameters={"tmid": 0.025, "rprs": 0.1, "inc": 89.0, "a2": 0.0}) + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 0.05, 21) + tflux = np.full(times.shape[0], 2.0) + cflux = np.full(times.shape[0], 2.0) + airmass = np.linspace(1.0, 1.5, times.shape[0]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + + myfit, _, _ = fit_lightcurve( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times, + final_fit_mode="ns", + ) + + assert myfit is not None + assert captured_calls[0]["mode"] == "lm" + assert captured_calls[1]["mode"] == "ns" + assert captured_calls[0]["bounds"]["tmid"] == pytest.approx([0.011347361950458953, 0.03865263804954105]) + assert captured_calls[1]["bounds"]["tmid"] == pytest.approx([0.011347361950458953, 0.03865263804954105]) + assert myfit.nested_tmid_refinement_applied is False + assert "one side of the modeled transit" in myfit.nested_tmid_refinement_note + + +def test_run_target_driven_photometry_search_selects_best_method_across_psf_and_aperture(monkeypatch): + evaluated = [] + + class DummyFit: + def __init__(self, residual_level, delta_bic, ktmf_metric): + self.residuals = np.full(6, residual_level) + self.data = np.ones(6) + self.transit_qc_delta_bic = delta_bic + self.transit_qc_ktmf_metric = ktmf_metric + + def fake_evaluate(task): + _, tflux, cflux, *_ = task + evaluated.append(np.asarray(cflux)) + cflux = np.asarray(cflux) + tflux = np.asarray(tflux) + if np.allclose(cflux, 20.0): + return { + "myfit": DummyFit(0.02, 9.0, 2.80), + "res_std": 0.02, + "transit_delta_bic": 9.0, + "ktmf_metric": 2.80, + }, tflux, cflux + if np.allclose(cflux, 40.0): + return { + "myfit": DummyFit(0.01, 18.0, 4.85), + "res_std": 0.01, + "transit_delta_bic": 18.0, + "ktmf_metric": 4.85, + }, tflux, cflux + raise AssertionError("Unexpected candidate flux passed to evaluator.") + + monkeypatch.setattr("exotic.exotic.evaluate_lightcurve_candidate", fake_evaluate) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.5, 6) + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + psf_target_amp = 20.0 / (2.0 * np.pi) + psf_comp_amp = 20.0 / (2.0 * np.pi) + psf_data = { + "target": np.column_stack([ + np.zeros(6), + np.zeros(6), + np.full(6, psf_target_amp), + np.ones(6), + np.ones(6), + ]), + "comp1": np.column_stack([ + np.ones(6), + np.ones(6), + np.full(6, psf_comp_amp), + np.ones(6), + np.ones(6), + ]), + } + aper_data = { + "target": np.full((6, 1, 1), 40.0), + "comp1": np.full((6, 1, 1), 40.0), + } + + result = run_target_driven_photometry_search( + times, + jd_times, + airmass, + ld, + p_dict, + comp_stars=[[100.0, 200.0]], + psf_data=psf_data, + aper_data=aper_data, + apers=np.array([5.0]), + annuli=np.array([12.0]), + sigma=1.0, + require_comp_star=True, + use_psf_photometry=True, + use_aperture_photometry=True, + ) + + assert len(evaluated) == 2 + assert {tuple(np.unique(values)) for values in evaluated} == {(20.0,), (40.0,)} + assert result["selection_metric"] == "ktmf" + assert result["best_candidate"]["method"] == "aperture" + assert result["best_candidate"]["comp_index"] == 0 + assert result["selected_ktmf_metric"] == pytest.approx(4.85) + assert result["selected_transit_delta_bic"] == pytest.approx(18.0) + + +def test_apply_raw_target_photometry_selection_sets_no_comparison_aperture_sentinel(): + fit = types.SimpleNamespace(time=np.linspace(0.0, 0.05, 6)) + target_flux = np.linspace(1000.0, 1010.0, 6) + target_driven_search = { + "best_candidate": { + "method": "aperture", + "a": 1, + "an": 2, + "aper": 5.0, + "annulus": 12.0, + "comp_index": None, + }, + "best_fit_lc": fit, + "selected_ktmf_metric": 3.5, + "selected_transit_delta_bic": 12.0, + "selection_metric": "ktmf", + "selected_eebls_snr": 7.0, + "flux_tar": target_flux, + "flux_ref": np.ones(6), + "selected_source_indices": np.arange(6), + "candidate_summaries": [{"selected": True, "fit_point_count": 6}], + } + photometry_info = {"min_aperture": None, "comp_star_num": None} + flux_values = {} + centroid_positions = {} + psf_data = { + "target": np.column_stack([ + np.linspace(10.0, 15.0, 6), + np.linspace(20.0, 25.0, 6), + ]) + } + + applied = apply_raw_target_photometry_selection( + target_driven_search, + photometry_info, + flux_values, + centroid_positions, + psf_data, + ) + + assert applied is True + assert photometry_info["best_fit_lc"] is fit + assert photometry_info["comp_star_num"] is None + assert photometry_info["min_aperture"] == pytest.approx(-5.0) + assert photometry_info["min_annulus"] == pytest.approx(12.0) + assert photometry_info["selection_basis"] == "raw_target_flux_fallback" + assert flux_values["flux_tar"] == pytest.approx(target_flux) + assert flux_values["flux_ref"] == pytest.approx(np.ones(6)) + assert flux_values["flux_unc_ref"] == pytest.approx(np.zeros(6)) + assert np.isnan(centroid_positions["x_ref"]).all() + assert np.isnan(centroid_positions["y_ref"]).all() + + +def test_run_target_driven_photometry_search_can_select_raw_target_without_comparison(monkeypatch): + times = np.linspace(0.0, 0.05, 6) + target_flux = np.linspace(1000.0, 1010.0, 6) + + def fake_evaluate(task): + candidate_times, candidate_target_flux, candidate_reference_flux, *_ = task + fit = types.SimpleNamespace( + time=np.asarray(candidate_times, dtype=float), + residuals=np.full(6, 0.01), + data=np.ones(6), + ) + return { + "myfit": fit, + "accepted": True, + "ktmf_metric": 3.0, + "fit_point_count": 6, + }, np.asarray(candidate_target_flux), np.asarray(candidate_reference_flux) + + monkeypatch.setattr("exotic.exotic.evaluate_lightcurve_candidate", fake_evaluate) + psf_data = { + "target": np.column_stack([ + np.linspace(10.0, 15.0, 6), + np.linspace(20.0, 25.0, 6), + ]) + } + aper_data = {"target": target_flux.reshape(6, 1, 1)} + + result = run_target_driven_photometry_search( + times, + 2460000.0 + times, + np.linspace(1.0, 1.5, 6), + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={}, + comp_stars=[], + psf_data=psf_data, + aper_data=aper_data, + apers=np.array([5.0]), + annuli=np.array([12.0]), + sigma=1.0, + require_comp_star=False, + use_psf_photometry=False, + use_aperture_photometry=True, + ) + + assert result["best_candidate"]["comp_index"] is None + assert result["best_candidate"]["method"] == "aperture" + assert result["flux_tar"] == pytest.approx(target_flux) + assert result["flux_ref"] == pytest.approx(np.ones(6)) + assert result["selected_source_indices"] == pytest.approx(np.arange(6)) + + +def test_run_target_driven_photometry_search_can_prefer_highest_eebls_snr(monkeypatch): + class DummyFit: + def __init__(self, residual_level, eebls_snr, delta_bic): + self.residuals = np.full(6, residual_level) + self.data = np.ones(6) + self.eebls_diagnostic_depth_snr = eebls_snr + self.transit_qc_delta_bic = delta_bic + + def fake_evaluate(task): + _, tflux, cflux, *_ = task + cflux = np.asarray(cflux, dtype=float) + tflux = np.asarray(tflux, dtype=float) + if np.allclose(cflux, 20.0): + return {"myfit": DummyFit(0.01, 4.0, 20.0), "res_std": 0.01, "eebls_snr": 4.0, "transit_delta_bic": 20.0}, tflux, cflux + if np.allclose(cflux, 40.0): + return {"myfit": DummyFit(0.02, 9.0, 12.0), "res_std": 0.02, "eebls_snr": 9.0, "transit_delta_bic": 12.0}, tflux, cflux + raise AssertionError("Unexpected candidate flux passed to evaluator.") + + monkeypatch.setattr("exotic.exotic.evaluate_lightcurve_candidate", fake_evaluate) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.5, 6) + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + psf_target_amp = 20.0 / (2.0 * np.pi) + psf_comp_amp = 20.0 / (2.0 * np.pi) + psf_data = { + "target": np.column_stack([ + np.zeros(6), + np.zeros(6), + np.full(6, psf_target_amp), + np.ones(6), + np.ones(6), + ]), + "comp1": np.column_stack([ + np.ones(6), + np.ones(6), + np.full(6, psf_comp_amp), + np.ones(6), + np.ones(6), + ]), + } + aper_data = { + "target": np.full((6, 1, 1), 40.0), + "comp1": np.full((6, 1, 1), 40.0), + } + + result = run_target_driven_photometry_search( + times, + jd_times, + airmass, + ld, + p_dict, + comp_stars=[[100.0, 200.0]], + psf_data=psf_data, + aper_data=aper_data, + apers=np.array([5.0]), + annuli=np.array([12.0]), + sigma=1.0, + require_comp_star=True, + use_psf_photometry=True, + use_aperture_photometry=True, + pick_comparison_by_eebls_snr=True, + ) + + assert result["selection_metric"] == "eebls_snr" + assert result["best_candidate"]["method"] == "aperture" + assert result["selected_eebls_snr"] == pytest.approx(9.0) + assert result["selected_transit_delta_bic"] == pytest.approx(12.0) + + +def test_fit_ranked_comparison_calibration_candidates_retries_next_best_candidate(monkeypatch): + class DummyFit: + def __init__(self): + self.residuals = np.full(6, 0.01) + self.data = np.ones(6) + + def fake_finalize(times, tflux, cflux, airmass, ld, p_dict, jd_times=None, **kwargs): + cflux = np.asarray(cflux, dtype=float) + if np.allclose(cflux, 0.0): + return { + "applied": False, + "fit": None, + "failure_reason": "the raw comparison-candidate photometry did not yield a usable light curve.", + "note": "test full reduction", + } + return { + "applied": True, + "fit": DummyFit(), + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": cflux, + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } + + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.5, 6) + comparison_calibration = { + "method": "aperture", + "a": 0, + "an": 0, + "aper": 5.0, + "annulus": 12.0, + "best_comp_index": 0, + "comp_summaries": [ + {"comp_index": 0, "key": "comp1", "aggregate_score": 0.01, "coverage_rejected": False}, + {"comp_index": 1, "key": "comp2", "aggregate_score": 0.02, "coverage_rejected": False}, + ], + } + aper_data = { + "target": np.full((6, 1, 1), 10.0), + "comp1": np.zeros((6, 1, 1)), + "comp2": np.full((6, 1, 1), 5.0), + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.ones(6), + ) + + assert [attempt["comp_index"] for attempt in result["attempts"]] == [0, 1] + assert result["selected_result"]["comp_index"] == 1 + assert result["attempts"][0]["fit"] is None + assert result["attempts"][0]["fit_diagnostics"]["failure_reason"] is not None + assert result["attempts"][1]["fit"] is not None + + +def test_fit_ranked_comparison_calibration_candidates_archives_qc_failed_run_and_tries_next(monkeypatch, tmp_path): + class DummyFit: + def __init__(self, qc_status): + self.residuals = np.full(6, 0.01) + self.data = np.ones(6) + self.transit_qc_status = qc_status + self.transit_qc_summary = ( + "Transit detection not supported strongly enough against a flat/null model (Delta BIC=2.50, Delta chi2=1.10)." + if qc_status == "fail" + else "Transit model strongly preferred over flat/null model (Delta BIC=18.40, Delta chi2=27.10)." + ) + self.transit_qc = {"status": qc_status, "summary": self.transit_qc_summary} + + def fake_finalize(times, tflux, cflux, airmass, ld, p_dict, jd_times=None, **kwargs): + cflux = np.asarray(cflux, dtype=float) + fit = DummyFit("fail" if np.allclose(cflux, 8.0) else "pass") + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": cflux, + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } + + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + def fake_save(save_dir, provisional_fit, final_fit, p_dict, observation_date, comp_index, **kwargs): + candidate_dir = Path(save_dir) / f"comp{comp_index + 1}" + candidate_dir.mkdir(parents=True, exist_ok=True) + return candidate_dir + + monkeypatch.setattr( + "exotic.exotic.save_comparison_candidate_full_reduction_outputs", + fake_save, + ) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.5, 6) + comparison_calibration = { + "method": "aperture", + "method_label": "Aperture photometry (aper=5.00px, annulus=12.00px)", + "a": 0, + "an": 0, + "aper": 5.0, + "annulus": 12.0, + "best_comp_index": 0, + "comp_summaries": [ + {"comp_index": 0, "key": "comp1", "label": "Comp 1", "position": [100.0, 200.0], "aggregate_score": 0.01, "coverage_rejected": False}, + {"comp_index": 1, "key": "comp2", "label": "Comp 2", "position": [300.0, 400.0], "aggregate_score": 0.02, "coverage_rejected": False}, + ], + } + aper_data = { + "target": np.full((6, 1, 1), 10.0), + "comp1": np.full((6, 1, 1), 8.0), + "comp2": np.full((6, 1, 1), 5.0), + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.ones(6), + save_dir=tmp_path, + planet_name="HAT-P-32 b", + observation_date="2026-04-28", + ) + + assert result["selected_result"]["comp_index"] == 1 + assert result["attempts"][0]["rejected_by_transit_qc"] is True + assert result["attempts"][0]["fit_diagnostics"]["failed_stage"] == "transit_qc" + failed_run_dir = result["attempts"][0]["failed_run_dir"] + assert failed_run_dir is not None + assert ( + tmp_path + / "Diagnostics" + / "comp_1_failed" + / "working_artifacts" + / "FailedFitSummary_HAT-P-32b_2026-04-28.json" + ).exists() + assert Path(failed_run_dir).exists() + + +def test_fit_ranked_comparison_calibration_candidates_falls_back_to_best_qc_rejected_fit( + monkeypatch, +): + class DummyFit: + def __init__(self, ktmf, delta_bic): + self.residuals = np.full(6, 0.01) + self.data = np.ones(6) + self.transit_qc_status = "fail" + self.transit_qc_summary = ( + "Transit model is preferred over the flat/null model, but QC rejected the fit because " + "the fit deviates too far from the expected published Rp/R* value " + f"(Delta BIC={delta_bic:.2f}, Delta chi2=27.10)." + ) + self.transit_qc = { + "status": "fail", + "summary": self.transit_qc_summary, + "ktmf_metric": ktmf, + "delta_bic": delta_bic, + } + self.transit_qc_ktmf_metric = ktmf + self.transit_qc_delta_bic = delta_bic + + def fake_finalize(times, tflux, cflux, airmass, ld, p_dict, jd_times=None, **kwargs): + cflux = np.asarray(cflux, dtype=float) + comp_marker = int(np.nanmedian(cflux)) + fit = DummyFit( + ktmf={8: 3.10, 5: 4.80}[comp_marker], + delta_bic={8: 18.0, 5: 30.0}[comp_marker], + ) + return { + "applied": True, + "fit": fit, + "good_target_flux": np.asarray(tflux, dtype=float), + "good_comp_flux": cflux, + "source_indices": np.arange(len(times), dtype=int), + "duration_samples": np.array([], dtype=float), + "data_highres": None, + "note": "test full reduction", + } + + monkeypatch.setattr("exotic.exotic.finalize_comparison_candidate_full_reduction", fake_finalize) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.5, 6) + comparison_calibration = { + "method": "aperture", + "method_label": "Aperture photometry (aper=5.00px, annulus=12.00px)", + "a": 0, + "an": 0, + "aper": 5.0, + "annulus": 12.0, + "best_comp_index": 0, + "comp_summaries": [ + {"comp_index": 0, "key": "comp1", "label": "Comp 1", "aggregate_score": 0.01, "coverage_rejected": False}, + {"comp_index": 1, "key": "comp2", "label": "Comp 2", "aggregate_score": 0.02, "coverage_rejected": False}, + ], + } + aper_data = { + "target": np.full((6, 1, 1), 10.0), + "comp1": np.full((6, 1, 1), 8.0), + "comp2": np.full((6, 1, 1), 5.0), + } + + result = fit_ranked_comparison_calibration_candidates( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={}, + comparison_calibration=comparison_calibration, + psf_data={}, + aper_data=aper_data, + target_psf_flux=np.ones(6), + ) + + assert [attempt["rejected_by_transit_qc"] for attempt in result["attempts"]] == [True, True] + assert result["selection_metric"] == "ktmf_combined_quality" + assert result["selected_result"]["comp_index"] == 1 + assert result["selected_result"]["selected_despite_transit_qc"] is True + assert result["selected_result"]["ktmf_metric"] == pytest.approx(4.80) + assert "best available fallback" in result["selected_result"]["selection_reason"] + + +def test_run_target_driven_photometry_search_skips_qc_failed_candidate(monkeypatch): + class DummyFit: + def __init__(self, residual_level, delta_bic=np.nan): + self.residuals = np.full(6, residual_level) + self.data = np.ones(6) + self.transit_qc_delta_bic = delta_bic + + def fake_evaluate(task): + _, tflux, cflux, *_ = task + cflux = np.asarray(cflux, dtype=float) + tflux = np.asarray(tflux, dtype=float) + if np.allclose(cflux, 20.0): + return { + "myfit": DummyFit(0.005, 3.0), + "accepted": False, + "res_std": 0.005, + "eebls_snr": 7.0, + "transit_delta_bic": 3.0, + "transit_qc_status": "fail", + "transit_qc_summary": "Transit detection not supported strongly enough against a flat/null model.", + "rejected_by_transit_qc": True, + "fit_diagnostics": {"failed_stage": "transit_qc", "usable_point_count": 6}, + "failure_reason": "Transit detection not supported strongly enough against a flat/null model.", + "fit_point_count": 6, + }, tflux, cflux + if np.allclose(cflux, 40.0): + return { + "myfit": DummyFit(0.02, 14.0), + "accepted": True, + "res_std": 0.02, + "eebls_snr": 4.0, + "transit_delta_bic": 14.0, + "transit_qc_status": "pass", + "transit_qc_summary": "Transit model strongly preferred over flat/null model.", + "rejected_by_transit_qc": False, + "fit_diagnostics": {"usable_point_count": 6}, + "failure_reason": None, + "fit_point_count": 6, + }, tflux, cflux + raise AssertionError("Unexpected candidate flux passed to evaluator.") + + monkeypatch.setattr("exotic.exotic.evaluate_lightcurve_candidate", fake_evaluate) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.5, 6) + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + psf_target_amp = 20.0 / (2.0 * np.pi) + psf_comp_amp = 20.0 / (2.0 * np.pi) + psf_data = { + "target": np.column_stack([ + np.zeros(6), + np.zeros(6), + np.full(6, psf_target_amp), + np.ones(6), + np.ones(6), + ]), + "comp1": np.column_stack([ + np.ones(6), + np.ones(6), + np.full(6, psf_comp_amp), + np.ones(6), + np.ones(6), + ]), + } + aper_data = { + "target": np.full((6, 1, 1), 40.0), + "comp1": np.full((6, 1, 1), 40.0), + } + + result = run_target_driven_photometry_search( + times, + jd_times, + airmass, + ld, + p_dict, + comp_stars=[[100.0, 200.0]], + psf_data=psf_data, + aper_data=aper_data, + apers=np.array([5.0]), + annuli=np.array([12.0]), + sigma=1.0, + require_comp_star=True, + use_psf_photometry=True, + use_aperture_photometry=True, + ) + + assert len(result["candidate_summaries"]) == 2 + assert result["candidate_summaries"][0]["rejected_by_transit_qc"] is True + assert result["best_candidate"]["method"] == "aperture" + assert result["selected_transit_delta_bic"] == pytest.approx(14.0) + + +def test_diagnose_lightcurve_fit_inputs_allows_large_ratios(): + diagnostics = diagnose_lightcurve_fit_inputs( + np.linspace(0.0, 0.05, 6), + np.full(6, 30.0), + np.full(6, 10.0), + np.linspace(1.0, 1.5, 6), + ) + + assert diagnostics["failure_reason"] is None + assert diagnostics["relative_flux_point_count"] == 6 + assert diagnostics["usable_point_count"] >= 5 + + +def test_prepare_lightcurve_fit_input_series_normalizes_ratio_around_unity(): + times = np.linspace(0.0, 0.05, 6) + prepared = prepare_lightcurve_fit_input_series( + times, + np.full(6, 30.0), + np.full(6, 10.0), + np.linspace(1.0, 1.5, 6), + ) + + assert prepared["applied"] is True + assert np.nanmedian(prepared["debug_raw_ratio"]) == pytest.approx(3.0) + assert prepared["approximate_baseline_level"] == pytest.approx(3.0) + assert np.nanmedian(prepared["flux"]) == pytest.approx(1.0) + + +def test_prepare_lightcurve_fit_input_series_uses_per_star_flux_errors(monkeypatch): + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 0.05, 6) + target_flux = np.full(6, 400.0) + comp_flux = np.full(6, 100.0) + target_error = np.full(6, 20.0) + comp_error = np.full(6, 5.0) + + prepared = prepare_lightcurve_fit_input_series( + times, + target_flux, + comp_flux, + np.linspace(1.0, 1.5, 6), + target_flux_error=target_error, + comp_flux_error=comp_error, + ) + + propagated_relative_error = np.sqrt((20.0 / 100.0) ** 2 + (5.0 * 400.0 / 100.0 ** 2) ** 2) + assert prepared["applied"] is True + assert np.nanmedian(prepared["debug_relative_flux_error"]) == pytest.approx(propagated_relative_error) + assert np.nanmedian(prepared["unc"]) == pytest.approx(propagated_relative_error / 4.0) + assert np.allclose(prepared["target_flux_error"], target_error) + assert np.allclose(prepared["comp_flux_error"], comp_error) + + +def test_prepare_lightcurve_fit_input_series_scales_target_only_counts_to_max_exposure(monkeypatch): + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 0.05, 6) + target_flux = np.array([100.0, 200.0, 200.0, 200.0, 300.0, 300.0]) + comp_flux = np.ones(6) + exposure_times = np.array([30.0, 60.0, 60.0, 60.0, 60.0, 60.0]) + + prepared = prepare_lightcurve_fit_input_series( + times, + target_flux, + comp_flux, + np.linspace(1.0, 1.5, 6), + exposure_times_seconds=exposure_times, + gain_e_per_adu=2.0, + ) + + expected_flux = np.array([200.0, 200.0, 200.0, 200.0, 300.0, 300.0]) + expected_error = np.sqrt(target_flux / 2.0) * np.array([2.0, 1.0, 1.0, 1.0, 1.0, 1.0]) + + assert prepared["applied"] is True + assert prepared["debug_target_flux"] == pytest.approx(expected_flux) + assert prepared["target_flux"] == pytest.approx(expected_flux) + assert prepared["target_flux_error"] == pytest.approx(expected_error) + assert prepared["debug_relative_flux_error"] == pytest.approx(expected_error) + + +def test_compute_photometry_noise_budget_includes_optional_terms(): + config = { + "gain_e_per_adu": 2.0, + "read_noise_electrons": 4.0, + "dark_current_electrons_per_second_per_pixel": 0.1, + "flat_field_fractional_error": 0.01, + "telescope_aperture_m": 0.3, + "scintillation_coefficient": 0.09, + "elevation_m": 100.0, + "enabled_terms": ( + "source", + "sky_aperture", + "sky_estimate", + "read", + "dark", + "flat", + "scintillation", + ), + } + + budget = compute_photometry_noise_budget( + 10000.0, + 3.0, + 50.0, + 200.0, + exposure_s=60.0, + airmass=1.2, + noise_config=config, + ) + + assert budget["source"] == pytest.approx(np.sqrt(10000.0 / 2.0)) + assert budget["read"] == pytest.approx(np.sqrt(50.0 * (4.0 / 2.0) ** 2)) + assert budget["dark"] == pytest.approx(np.sqrt(50.0 * 0.1 * 60.0 / 2.0 ** 2)) + assert budget["flat"] == pytest.approx(100.0) + assert budget["scintillation"] > 0 + assert budget["total"] > budget["flat"] + + +def test_noise_budget_config_reads_inits_and_header_values(): + header = { + "GAIN": 99.0, + "EGAIN": 1.5, + "RDNOISE": 7.0, + "DARKCURR": 0.02, + "FLATERR": 0.003, + "APR-DIA": 250.0, + } + config = noise_budget_config_from_info( + { + "read_noise_electrons": 5.0, + }, + header=header, + ) + + assert config["gain_e_per_adu"] == pytest.approx(1.5) + assert config["read_noise_electrons"] == pytest.approx(5.0) + assert config["dark_current_electrons_per_second_per_pixel"] == pytest.approx(0.02) + assert config["flat_field_fractional_error"] == pytest.approx(0.003) + assert config["telescope_aperture_m"] == pytest.approx(0.25) + assert "read" in config["enabled_terms"] + assert "flat" in config["enabled_terms"] + + +def test_prepare_lightcurve_fit_input_series_clips_prefit_raw_ratio_outliers(monkeypatch): + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 0.08, 21) + comp_flux = np.full(times.shape, 1000.0, dtype=float) + raw_ratio = np.ones(times.shape, dtype=float) + raw_ratio[8:13] = 0.98 + raw_ratio[15] = 1.55 + raw_ratio[16] = 0.72 + target_flux = raw_ratio * comp_flux + + prepared = prepare_lightcurve_fit_input_series( + times, + target_flux, + comp_flux, + np.linspace(1.0, 1.4, times.shape[0]), + expected_transit_depth=0.02, + ) + + assert prepared["applied"] is True + assert prepared["initial_sigma_keep_mask"].all() + assert prepared["prefit_raw_ratio_keep_mask"].tolist()[15:17] == [False, False] + assert np.any(np.isclose(prepared["time"], times[10])) + assert not np.any(np.isclose(prepared["time"], times[15])) + assert any( + diagnostic["stage"] == "Pre-fit raw-ratio outlier clip" + and diagnostic["dropped_point_count"] == 2 + for diagnostic in prepared["filter_diagnostics"] + ) + + +def test_run_target_driven_photometry_search_returns_failed_candidate_summaries(monkeypatch): + monkeypatch.setattr("exotic.exotic.fit_lightcurve", lambda *args, **kwargs: (None, None, None)) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.5, 6) + aper_data = { + "target": np.full((6, 1, 1), 30.0), + "comp1": np.full((6, 1, 1), 10.0), + "comp2": np.full((6, 1, 1), 12.0), + } + + result = run_target_driven_photometry_search( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={}, + comp_stars=[[100.0, 200.0], [300.0, 400.0]], + psf_data={}, + aper_data=aper_data, + apers=np.array([7.05]), + annuli=np.array([22.73]), + sigma=1.0, + require_comp_star=True, + use_psf_photometry=False, + use_aperture_photometry=True, + multiprocess_lightcurve_fits=0, + ) + + assert result["best_candidate"] is None + assert len(result["candidate_summaries"]) == 2 + assert all( + summary["failure_reason"] is not None + for summary in result["candidate_summaries"] + ) + assert all( + ">2x=" not in summary["failure_reason"] + for summary in result["candidate_summaries"] + ) + assert result["candidate_summaries"][0]["method_label"] == "Aperture photometry (aper=7.05px, annulus=22.73px)" + + +def test_fit_lightcurve_to_every_comparison_candidate_forwards_full_plot_time_range(monkeypatch): + captured_plot_ranges = [] + + class DummyFit: + def __init__(self, plot_time_range): + self.plot_time_range = plot_time_range + self.parameters = {"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a0": 1.0, "a2": 0.0} + self.errors = {"tmid": 0.001, "rprs": 0.001, "inc": 0.1, "a0": 0.01, "a2": 0.01} + self.residuals = np.full(6, 0.01, dtype=float) + self.data = np.ones(6, dtype=float) + + def fake_fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times=None, **kwargs): + plot_time_range = kwargs.get("plot_time_range") + captured_plot_ranges.append(plot_time_range) + return DummyFit(plot_time_range), np.asarray(tflux, dtype=float), np.asarray(cflux, dtype=float) + + monkeypatch.setattr("exotic.exotic.fit_lightcurve", fake_fit_lightcurve) + monkeypatch.setattr( + "exotic.exotic.diagnose_lightcurve_fit_inputs", + lambda *args, **kwargs: { + "input_point_count": 6, + "has_reference_flux": True, + "relative_flux_point_count": 6, + "sigma_clip_point_count": 6, + "usable_point_count": 6, + "failed_stage": None, + "failure_reason": None, + }, + ) + + times = np.linspace(0.0, 0.05, 6) + jd_times = 2460000.0 + times + airmass = np.linspace(1.0, 1.5, 6) + psf_series = np.ones((6, 7), dtype=float) + psf_data = { + "target": psf_series.copy(), + "comp1": psf_series.copy(), + } + photometry_info = { + "best_fit_lc": object(), + "comp_star_num": 1, + "min_aperture": 0, + } + plot_time_range = (-0.12, 0.18) + + summaries = fit_lightcurve_to_every_comparison_candidate( + times, + jd_times, + airmass, + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={}, + comp_stars=[[100.0, 200.0]], + psf_data=psf_data, + aper_data=None, + photometry_info=photometry_info, + plot_time_range=plot_time_range, + ) + + assert captured_plot_ranges == [plot_time_range] + assert summaries[0]["fit"].plot_time_range == pytest.approx(plot_time_range) + + +def test_ensure_lightcurve_fit_failure_reason_preserves_existing_diagnostic_reason(): + diagnostics = { + "failed_stage": "minimum_points", + "failure_reason": "only 4 usable point(s) remained after filtering; need at least 5 for a lightcurve fit.", + } + + result = ensure_lightcurve_fit_failure_reason( + diagnostics, + fit_result=None, + failed_stage="lightcurve_fit", + failure_reason="the lightcurve fitter did not converge to a usable solution.", + ) + + assert result["failed_stage"] == "minimum_points" + assert result["failure_reason"] == diagnostics["failure_reason"] + + +def test_ensure_lightcurve_fit_failure_reason_adds_generic_reason_when_missing(): + diagnostics = { + "failed_stage": None, + "failure_reason": None, + } + + result = ensure_lightcurve_fit_failure_reason( + diagnostics, + fit_result=None, + failed_stage="lightcurve_fit", + failure_reason="the lightcurve fitter did not converge to a usable solution.", + ) + + assert result["failed_stage"] == "lightcurve_fit" + assert result["failure_reason"] == "the lightcurve fitter did not converge to a usable solution." + + +def test_fit_lightcurve_can_disable_impact_parameter_parameterization(monkeypatch): + captured = {"flags": []} + + def fake_lc_fitter( + times, + fluxes, + flux_unc, + airmass, + prior, + bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + captured["flags"].append(use_impactparameter_rather_than_inclination_to_fit) + return types.SimpleNamespace() + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 0.05, 6) + tflux = np.full(times.shape[0], 2.0) + cflux = np.full(times.shape[0], 2.0) + airmass = np.linspace(1.0, 1.5, times.shape[0]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + + fit_lightcurve( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times, + use_impactparameter_rather_than_inclination_to_fit=False, + ) + + assert captured["flags"] == [False] + + +def test_fit_lightcurve_forwards_exposure_times_to_fitter(monkeypatch): + captured = {} + + def fake_lc_fitter( + times, + fluxes, + flux_unc, + airmass, + prior, + bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + exposure_times_seconds=None, + ): + captured["times"] = np.asarray(times, dtype=float) + captured["exposure_times_seconds"] = None if exposure_times_seconds is None else np.asarray( + exposure_times_seconds, + dtype=float, + ) + return types.SimpleNamespace() + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 0.05, 6) + exposure_times = np.array([60.0, 60.0, 90.0, 90.0, 120.0, 120.0]) + tflux = np.full(times.shape[0], 2.0) + cflux = np.full(times.shape[0], 2.0) + airmass = np.linspace(1.0, 1.5, times.shape[0]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + + fit_lightcurve( + times, + tflux, + cflux, + airmass, + ld, + p_dict, + jd_times, + exposure_times_seconds=exposure_times, + ) + + assert captured["times"] == pytest.approx(times) + assert captured["exposure_times_seconds"] == pytest.approx(exposure_times) + + +def test_build_initial_ars_bounds_prefers_published_uncertainty_when_available(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_ENABLED", True) + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_PERCENTAGE", 10.0) + + assert build_initial_ars_bounds(15.0, 0.1) == pytest.approx([13.5, 16.5]) + assert build_initial_ars_bounds(15.0, None) == pytest.approx([11.25, 18.75]) + + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_ENABLED", False) + assert build_initial_ars_bounds(15.0, None) == pytest.approx([11.25, 18.75]) + + +def test_build_single_transit_duration_prior_uses_published_geometry_uncertainties(): + duration_prior = build_single_transit_duration_prior({ + "pPer": 1.0, + "pPerUnc": 0.001, + "rprs": 0.1, + "rprsUnc": 0.01, + "aRs": 15.0, + "aRsUnc": 0.1, + "inc": 89.0, + "incUnc": 0.1, + "ecc": 0.0, + "omega": 0.0, + }) + + assert duration_prior["applied"] is True + assert duration_prior["expected_duration"] > 0 + assert duration_prior["sigma_log_duration"] > 0 + assert duration_prior["relative_sigma"] >= 0.049 + assert duration_prior["source"] == "published geometry uncertainties" + assert "published geometry uncertainties" in duration_prior["note"] + + +def test_fit_lightcurve_skips_airmass_term_when_airmass_span_is_small(monkeypatch): + captured = {} + + def fake_lc_fitter( + times, + fluxes, + flux_unc, + airmass, + prior, + bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ): + captured["bounds"] = dict(bounds) + captured["airmass"] = np.array(airmass) + return types.SimpleNamespace() + + monkeypatch.setattr("exotic.exotic.lc_fitter", fake_lc_fitter) + monkeypatch.setattr( + "exotic.exotic.sigma_clip", + lambda data, sigma=3, dt=21, po=2, times=None: np.zeros(len(data), dtype=bool), + ) + + times = np.linspace(0.0, 0.05, 6) + tflux = np.full(times.shape[0], 2.0) + cflux = np.full(times.shape[0], 2.0) + airmass = np.array([1.10, 1.11, 1.12, 1.13, 1.14, 1.15]) + jd_times = 2460000.0 + times + ld = [0.1, 0.1, 0.1, 0.1] + p_dict = { + "rprs": 0.1, + "aRs": 15.0, + "aRsUnc": 0.1, + "pPer": 1.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + "midT": 0.02, + "midTUnc": 0.001, + "pPerUnc": 0.001, + } + + myfit, _, _ = fit_lightcurve(times, tflux, cflux, airmass, ld, p_dict, jd_times) + + assert myfit is not None + assert list(captured["bounds"])[:4] == ["rprs", "tmid", "ars", "inc"] + assert captured["bounds"]["ars"] == pytest.approx([13.5, 16.5]) + assert "a2" not in captured["bounds"] + assert myfit.airmass_fit_skipped is True + + +def _run_main_until_vertical_flux_bound( + monkeypatch, + tmp_path, + disable_vertical_flux_normalization=Ellipsis, + random_seed=123, + override=True, + nasa_result=None, + target_ra=10.0, + target_dec=20.0, + ephemeris_overrides=None, + expected_ephemeris=None, + expected_error=None): + import exotic.exotic as exotic_module + + class BoundReached(Exception): + pass + + prered_file = tmp_path / "prereduced.csv" + prered_file.write_text( + "\n".join( + [ + "2450000.00,1.00,0.01,1.10", + "2450000.10,1.01,0.01,1.12", + "2450000.20,0.99,0.01,1.14", + "2450000.30,1.00,0.01,1.16", + "2450000.40,1.02,0.01,1.18", + "2450000.50,1.01,0.01,1.20", + ] + ) + ) + + user_pdict = { + "ra": target_ra, + "dec": target_dec, + "pName": "Test Planet b", + "sName": "Test Star", + "pPer": 1.0, + "pPerUnc": 0.001, + "midT": 2450000.25, + "midTUnc": 0.001, + "rprs": 0.1, + "rprsUnc": 0.01, + "aRs": 15.0, + "aRsUnc": 0.1, + "inc": 89.0, + "incUnc": 0.1, + "omega": 0.0, + "ecc": 0.0, + "teff": 5500.0, + "teffUncPos": 100.0, + "teffUncNeg": 100.0, + "met": 0.0, + "metUncPos": 0.1, + "metUncNeg": 0.1, + "logg": 4.4, + "loggUncPos": 0.1, + "loggUncNeg": 0.1, + "dist": 100.0, + "pm_ra": 0.0, + "pm_dec": 0.0, + } + if ephemeris_overrides: + user_pdict.update(ephemeris_overrides) + exotic_info = { + "save": tmp_path, + "prered_file": prered_file, + "file_time": "BJD_TDB", + "file_units": "flux", + "airmass_already_corrected": False, + "random_seed": random_seed, + "date": "2026-03-19", + } + if disable_vertical_flux_normalization is not Ellipsis: + exotic_info["disable_vertical_flux_normalization"] = disable_vertical_flux_normalization + + args = types.SimpleNamespace( + multiprocess_transformations=None, + multiprocess_lightcurve_fits=None, + realtime=None, + reduce=None, + prereduced=str(tmp_path / "inits.json"), + photometry=None, + override=override, + nasaexoarch=False, + non_interactive_run=True, + use_nextastro_astrometry=False, + use_nextastro_variability_server=False, + ) + + class FakeInputs: + def __init__(self, init_opt): + self.init_opt = init_opt + + def search_init(self, init_path, planet_dict): + return init_path, dict(user_pdict) + + def prereduced(self, planet): + return dict(exotic_info), planet or user_pdict["pName"] + + captured = {} + + monkeypatch.setattr(exotic_module, "parse_args", lambda: args) + monkeypatch.setattr(exotic_module, "Inputs", FakeInputs) + if nasa_result is not None: + class FakeNASAExoplanetArchive: + def __init__(self, planet, non_interactive=False): + self.planet = planet + self.non_interactive = non_interactive + + def planet_info(self): + return nasa_result + + monkeypatch.setattr(exotic_module, "NASAExoplanetArchive", FakeNASAExoplanetArchive) + monkeypatch.setattr( + exotic_module, + "get_ld_values", + lambda *_args, **_kwargs: ([0.1, 0.1, 0.1, 0.1], [0.1], [0.1], [0.1], [0.1]), + ) + + def fake_apply_vertical_flux_normalization_bound(prior, bounds, flux_values, disabled): + captured["disabled"] = disabled + if expected_ephemeris is not None: + assert prior['per'] == pytest.approx(expected_ephemeris['pPer']) + assert prior['tmid'] == pytest.approx(expected_ephemeris['midT']) + raise BoundReached() + + monkeypatch.setattr( + exotic_module, + "apply_vertical_flux_normalization_bound", + fake_apply_vertical_flux_normalization_bound, + ) + + if expected_error is not None: + with pytest.raises(ValueError, match=expected_error): + exotic_module.main() + return None + + with pytest.raises(BoundReached): + exotic_module.main() + + return captured["disabled"] + + +def test_main_prereduced_defaults_vertical_flux_normalization_to_enabled(monkeypatch, tmp_path): + disabled = _run_main_until_vertical_flux_bound(monkeypatch, tmp_path) + + assert disabled is False + + +def test_main_prereduced_respects_disable_vertical_flux_normalization_option(monkeypatch, tmp_path): + disabled = _run_main_until_vertical_flux_bound(monkeypatch, tmp_path, disable_vertical_flux_normalization=True) + + assert disabled is True + + +def test_main_prereduced_override_invalid_coordinates_use_nasa_fallback_without_prompt(monkeypatch, tmp_path): + monkeypatch.setattr( + 'builtins.input', + lambda prompt: pytest.fail("non-interactive coordinate resolution must not prompt"), + ) + + disabled = _run_main_until_vertical_flux_bound( + monkeypatch, + tmp_path, + override=True, + nasa_result=("Test Planet b", False, {"ra": 123.456, "dec": -45.678}), + target_ra="not-an-ra", + target_dec="not-a-dec", + ) + + assert disabled is False + + +def test_main_prereduced_override_missing_ephemeris_uses_nasa_fallback(monkeypatch, tmp_path): + archive_parameters = { + 'pPer': 2.5, + 'pPerUnc': 0.001, + 'midT': 2450000.25, + 'midTUnc': 0.002, + } + + disabled = _run_main_until_vertical_flux_bound( + monkeypatch, + tmp_path, + override=True, + nasa_result=("Test Planet b", False, archive_parameters), + ephemeris_overrides={'pPer': None, 'midT': 0.0}, + expected_ephemeris=archive_parameters, + ) + + assert disabled is False + + +def test_main_prereduced_stops_before_fitting_when_required_ephemeris_cannot_be_resolved( + monkeypatch, tmp_path): + _run_main_until_vertical_flux_bound( + monkeypatch, + tmp_path, + override=True, + nasa_result=("Test Planet b", False, {'pPer': np.nan, 'midT': None}), + ephemeris_overrides={'pPer': None, 'midT': 0.0}, + expected_error=r"Cannot start EXOTIC reduction.*pPer.*midT", + ) + + +def test_main_prereduced_generates_seed_after_candidate_falls_back_to_inits(monkeypatch, tmp_path): + disabled = _run_main_until_vertical_flux_bound( + monkeypatch, + tmp_path, + random_seed=None, + override=False, + nasa_result=("TOI-3514.01", True, None), + ) + + assert disabled is False + + +def test_cli_logs_unhandled_exception_once(monkeypatch): + import exotic.exotic as exotic_module + + logged = [] + + monkeypatch.setattr(exotic_module, "configure_runtime_logging", lambda *args, **kwargs: None) + monkeypatch.setattr(exotic_module, "install_exception_hooks", lambda: None) + monkeypatch.setattr(exotic_module, "main", lambda: (_ for _ in ()).throw(RuntimeError("boom"))) + + def fake_log_exception(message, exc_type, exc_value, exc_traceback): + logged.append((message, exc_type, str(exc_value), exc_traceback is not None)) + + monkeypatch.setattr(exotic_module, "_log_exception_with_fallback", fake_log_exception) + + with pytest.raises(RuntimeError, match="boom"): + exotic_module.cli() + + assert logged == [("Unhandled exception during EXOTIC run", RuntimeError, "boom", True)] + + +def test_package_init_exports_lazy_main_and_cli(monkeypatch): + import exotic + + monkeypatch.setattr(exotic, "_load_runtime_callable", lambda name: lambda: name) + + assert exotic.main() == "main" + assert exotic.cli() == "cli" + + +def test_package_init_loads_nested_runtime_for_archive_layout(monkeypatch): + import exotic + + def fake_import_module(module_name): + if module_name == "exotic.exotic.exotic": + return types.SimpleNamespace(main=lambda: "nested-main") + raise AssertionError(f"unexpected import: {module_name}") + + monkeypatch.setitem(exotic.__dict__, "__name__", "exotic.exotic") + monkeypatch.setattr(exotic, "import_module", fake_import_module) + + assert exotic._load_runtime_callable("main")() == "nested-main" + + +def test_configure_runtime_logging_rebinds_console_handler_to_current_stdout(monkeypatch, tmp_path): + import io + import exotic.exotic as exotic_module + + original_handlers = list(exotic_module.log.handlers) + original_configured = exotic_module._RUNTIME_LOGGING_CONFIGURED + original_basename = exotic_module._RUNTIME_LOG_BASENAME + original_path = exotic_module._RUNTIME_LOG_PATH + + try: + exotic_module.log.handlers = [] + exotic_module._RUNTIME_LOGGING_CONFIGURED = False + exotic_module._RUNTIME_LOG_BASENAME = None + exotic_module._RUNTIME_LOG_PATH = None + monkeypatch.setattr(exotic_module, "_reset_runtime_traceback_watchdog", lambda: None) + + first_stdout = io.StringIO() + monkeypatch.setattr(exotic_module.sys, "stdout", first_stdout) + exotic_module.configure_runtime_logging(output_dir=tmp_path, start_new_run=True) + handler = exotic_module._find_runtime_handler(exotic_module._RUNTIME_CONSOLE_HANDLER_NAME) + assert handler.stream is first_stdout + + second_stdout = io.StringIO() + monkeypatch.setattr(exotic_module.sys, "stdout", second_stdout) + exotic_module.configure_runtime_logging(output_dir=tmp_path) + assert handler.stream is second_stdout + finally: + exotic_module._close_runtime_file_handler() + exotic_module.log.handlers = original_handlers + exotic_module._RUNTIME_LOGGING_CONFIGURED = original_configured + exotic_module._RUNTIME_LOG_BASENAME = original_basename + exotic_module._RUNTIME_LOG_PATH = original_path + + +def test_configure_runtime_logging_does_not_use_environment_root_handlers(monkeypatch, tmp_path, capsys): + import io + import logging + import exotic.exotic as exotic_module + + class DisconnectedColabStream(io.StringIO): + def write(self, _value): + raise OSError(107, "Transport endpoint is not connected") + + def flush(self): + raise OSError(107, "Transport endpoint is not connected") + + original_handlers = list(exotic_module.log.handlers) + original_propagate = exotic_module.log.propagate + original_configured = exotic_module._RUNTIME_LOGGING_CONFIGURED + original_basename = exotic_module._RUNTIME_LOG_BASENAME + original_path = exotic_module._RUNTIME_LOG_PATH + root_logger = logging.getLogger() + original_root_handlers = list(root_logger.handlers) + original_root_level = root_logger.level + + try: + exotic_module.log.handlers = [] + exotic_module.log.propagate = True + exotic_module._RUNTIME_LOGGING_CONFIGURED = False + exotic_module._RUNTIME_LOG_BASENAME = None + exotic_module._RUNTIME_LOG_PATH = None + root_logger.handlers = [logging.StreamHandler(DisconnectedColabStream())] + root_logger.setLevel(logging.WARNING) + monkeypatch.setattr(exotic_module, "_reset_runtime_traceback_watchdog", lambda: None) + + exotic_module.configure_runtime_logging(output_dir=tmp_path, start_new_run=True) + exotic_module.log.debug("frame progress written only to EXOTIC's file handler") + + assert exotic_module.log.propagate is False + assert root_logger.level == logging.WARNING + assert "Logging error" not in capsys.readouterr().err + finally: + exotic_module._close_runtime_file_handler() + exotic_module.log.handlers = original_handlers + exotic_module.log.propagate = original_propagate + exotic_module._RUNTIME_LOGGING_CONFIGURED = original_configured + exotic_module._RUNTIME_LOG_BASENAME = original_basename + exotic_module._RUNTIME_LOG_PATH = original_path + root_logger.handlers = original_root_handlers + root_logger.setLevel(original_root_level) + + +def test_runtime_file_handler_suppresses_disconnected_mount_and_reopens(monkeypatch, tmp_path, capsys): + import io + import logging + import exotic.exotic as exotic_module + + class DisconnectedDriveStream(io.StringIO): + def write(self, _value): + raise OSError(107, "Transport endpoint is not connected") + + def flush(self): + raise OSError(107, "Transport endpoint is not connected") + + def close(self): + pass + + log_path = tmp_path / "EXOTIC_RunLog_test.log" + handler = exotic_module.FailSoftRuntimeFileHandler(log_path, mode="a", encoding="utf-8") + handler.setFormatter(logging.Formatter("%(message)s")) + handler.stream = DisconnectedDriveStream() + recovered_stream = io.StringIO() + monkeypatch.setattr(handler, "_open", lambda: recovered_stream) + + try: + handler.emit(logging.LogRecord("exotic", logging.DEBUG, __file__, 1, "frame 18", (), None)) + first_output = capsys.readouterr() + assert "Logging error" not in first_output.err + assert "run log stream disconnected" in first_output.out + assert handler.stream is None + + handler.emit(logging.LogRecord("exotic", logging.DEBUG, __file__, 1, "frame 19", (), None)) + second_output = capsys.readouterr() + assert "Logging error" not in second_output.err + assert "run log stream disconnected" not in second_output.out + assert recovered_stream.getvalue() == "frame 19\n" + finally: + handler.stream = None + handler.close() + + +def test_runtime_output_directory_is_read_from_command_line_init_file(tmp_path): + import exotic.exotic as exotic_module + + output_dir = tmp_path / "run output" + init_path = tmp_path / "inits.json" + init_path.write_text(json.dumps({ + "user_info": {"Directory to Save Plots": str(output_dir)}, + }), encoding="utf-8") + + assert exotic_module._runtime_output_directory_from_command_line( + ["-red", str(init_path), "-ov"] + ) == str(output_dir) + assert exotic_module._runtime_output_directory_from_command_line( + [f"--reduce={init_path}"] + ) == str(output_dir) + + +def test_runtime_logging_relocates_startup_content_and_keeps_runs_unique(monkeypatch, tmp_path): + import exotic.exotic as exotic_module + + original_handlers = list(exotic_module.log.handlers) + original_configured = exotic_module._RUNTIME_LOGGING_CONFIGURED + original_basename = exotic_module._RUNTIME_LOG_BASENAME + original_path = exotic_module._RUNTIME_LOG_PATH + + try: + exotic_module.log.handlers = [] + exotic_module._RUNTIME_LOGGING_CONFIGURED = False + exotic_module._RUNTIME_LOG_BASENAME = None + exotic_module._RUNTIME_LOG_PATH = None + monkeypatch.setattr(exotic_module.tempfile, "gettempdir", lambda: str(tmp_path / "staging")) + monkeypatch.setattr(exotic_module, "_reset_runtime_traceback_watchdog", lambda: None) + + exotic_module.configure_runtime_logging(start_new_run=True) + staged_log = Path(exotic_module._RUNTIME_LOG_PATH) + exotic_module.log_info("startup message before the save directory was known") + + output_dir = tmp_path / "output" + exotic_module.configure_runtime_logging(output_dir=output_dir) + first_log = Path(exotic_module._RUNTIME_LOG_PATH) + exotic_module.log_info("message after the save directory was known") + exotic_module.close_runtime_logging() + + assert not staged_log.exists() + assert first_log.parent == output_dir.resolve() / "Diagnostics" + first_content = first_log.read_text(encoding="utf-8") + assert "startup message before the save directory was known" in first_content + assert "message after the save directory was known" in first_content + + exotic_module.configure_runtime_logging(output_dir=output_dir, start_new_run=True) + second_log = Path(exotic_module._RUNTIME_LOG_PATH) + exotic_module.log_info("second run message") + exotic_module.close_runtime_logging() + + assert second_log != first_log + assert len(list((output_dir / "Diagnostics").glob("EXOTIC_RunLog_*.log"))) == 2 + assert "second run message" in second_log.read_text(encoding="utf-8") + finally: + exotic_module._close_runtime_file_handler() + exotic_module.log.handlers = original_handlers + exotic_module._RUNTIME_LOGGING_CONFIGURED = original_configured + exotic_module._RUNTIME_LOG_BASENAME = original_basename + exotic_module._RUNTIME_LOG_PATH = original_path + + +def test_log_exception_with_fallback_writes_traceback_to_current_stdout(monkeypatch, capsys): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "_logger_has_current_stdout_handler", lambda logger: False) + + try: + raise RuntimeError("boom") + except RuntimeError as exc: + exotic_module._log_exception_with_fallback( + "Unhandled exception during EXOTIC run", + type(exc), + exc, + exc.__traceback__, + ) + + output = capsys.readouterr().out + assert "Unhandled exception during EXOTIC run" in output + assert "Traceback" in output + assert "RuntimeError: boom" in output + + +def test_main_logs_direct_call_exceptions_to_current_stdout(monkeypatch, capsys): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "configure_runtime_logging", lambda *args, **kwargs: None) + monkeypatch.setattr(exotic_module, "install_exception_hooks", lambda: None) + monkeypatch.setattr(exotic_module, "_logger_has_current_stdout_handler", lambda logger: False) + monkeypatch.setattr(exotic_module, "_main_impl", lambda: (_ for _ in ()).throw(RuntimeError("boom"))) + + with pytest.raises(RuntimeError, match="boom"): + exotic_module.main() + + output = capsys.readouterr().out + assert "Unhandled exception during EXOTIC run" in output + assert "RuntimeError: boom" in output + + +def test_main_suppresses_all_internal_logging_error_tracebacks(monkeypatch, capsys): + import io + import logging + import exotic.exotic as exotic_module + + class DisconnectedColabStream(io.StringIO): + def write(self, _value): + raise OSError(107, "Transport endpoint is not connected") + + def flush(self): + raise OSError(107, "Transport endpoint is not connected") + + environment_logger = logging.getLogger("test.disconnected_colab_handler") + environment_logger.handlers = [logging.StreamHandler(DisconnectedColabStream())] + environment_logger.propagate = False + original_raise_exceptions = logging.raiseExceptions + + monkeypatch.setattr(exotic_module, "configure_runtime_logging", lambda *args, **kwargs: None) + monkeypatch.setattr(exotic_module, "install_exception_hooks", lambda: None) + monkeypatch.setattr(exotic_module, "cancel_runtime_traceback_watchdog", lambda: None) + monkeypatch.setattr(exotic_module, "close_runtime_logging", lambda: None) + + def report_through_disconnected_handler(): + environment_logger.error("frame progress") + return "completed" + + monkeypatch.setattr(exotic_module, "_main_impl", report_through_disconnected_handler) + + try: + logging.raiseExceptions = True + assert exotic_module.main() == "completed" + output = capsys.readouterr() + assert "--- Logging error ---" not in output.err + assert "Transport endpoint is not connected" not in output.err + assert logging.raiseExceptions is True + finally: + environment_logger.handlers = [] + logging.raiseExceptions = original_raise_exceptions diff --git a/tests/test_exotic_rprs_retry.py b/tests/test_exotic_rprs_retry.py new file mode 100644 index 00000000..a298aae6 --- /dev/null +++ b/tests/test_exotic_rprs_retry.py @@ -0,0 +1,2637 @@ +import importlib.util +import sys +import types + +import numpy as np +import pytest + + +def _module_available(name: str) -> bool: + try: + return importlib.util.find_spec(name) is not None + except (ModuleNotFoundError, ValueError): + return False + + +def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: + if not _module_available(name): + sys.modules.setdefault(name, module) + + +fake_barycorrpy = types.ModuleType("barycorrpy") +fake_utc_tdb = types.ModuleType("barycorrpy.utc_tdb") +fake_utc_tdb.JDUTC_to_BJDTDB = lambda *args, **kwargs: None +fake_astroalign = types.ModuleType("astroalign") +fake_astroalign.PIXEL_TOL = 1 +fake_astroquery = types.ModuleType("astroquery") +fake_astroquery_simbad = types.ModuleType("astroquery.simbad") +fake_astroquery_simbad.Simbad = type("Simbad", (), {}) +fake_astroquery_gaia = types.ModuleType("astroquery.gaia") +fake_astroquery_gaia.Gaia = type("Gaia", (), {}) +fake_imreg_dft = types.ModuleType("imreg_dft") +fake_colour_demosaicing = types.ModuleType("colour_demosaicing") +fake_colour_demosaicing.demosaicing_CFA_Bayer_bilinear = lambda *args, **kwargs: None +fake_photutils = types.ModuleType("photutils") +fake_photutils_aperture = types.ModuleType("photutils.aperture") +fake_photutils_aperture.CircularAperture = type("CircularAperture", (), {}) +fake_photutils_aperture.CircularAnnulus = type("CircularAnnulus", (), {}) +fake_photutils_detection = types.ModuleType("photutils.detection") +fake_photutils_detection.DAOStarFinder = type("DAOStarFinder", (), {}) +fake_ldtk = types.ModuleType("ldtk") +fake_ldtk.LDPSet = type("LDPSet", (), {}) +fake_ldtk.ldtk = types.SimpleNamespace(LDPSet=fake_ldtk.LDPSet) +fake_ldtk_ldmodel = types.ModuleType("ldtk.ldmodel") +fake_ldtk_ldmodel.LinearModel = type("LinearModel", (), {}) +fake_ldtk_ldmodel.QuadraticModel = type("QuadraticModel", (), {}) +fake_ldtk_ldmodel.NonlinearModel = type("NonlinearModel", (), {}) +fake_lmfit = types.ModuleType("lmfit") +fake_pylightcurve = types.ModuleType("pylightcurve") +fake_pylightcurve_models = types.ModuleType("pylightcurve.models") +fake_pylightcurve_exoplanet = types.ModuleType("pylightcurve.models.exoplanet_lc") +fake_pylightcurve_exoplanet.transit = lambda *args, **kwargs: None +fake_pyvo = types.ModuleType("pyvo") +fake_ultranest = types.ModuleType("ultranest") +fake_ultranest.ReactiveNestedSampler = type("ReactiveNestedSampler", (), {}) +fake_elca = types.ModuleType("exotic.api.elca") +fake_elca.lc_fitter = lambda *args, **kwargs: None +fake_elca.binner = lambda *args, **kwargs: None +fake_elca.transit = lambda *args, **kwargs: None +fake_elca.get_phase = lambda *args, **kwargs: None +fake_ld = types.ModuleType("exotic.api.ld") +fake_ld.LimbDarkening = type("LimbDarkening", (), {}) +fake_ld.ld_re_punct_p = lambda *args, **kwargs: None + +_set_stub_if_missing("astroalign", fake_astroalign) +_set_stub_if_missing("astroquery", fake_astroquery) +_set_stub_if_missing("astroquery.simbad", fake_astroquery_simbad) +_set_stub_if_missing("astroquery.gaia", fake_astroquery_gaia) +_set_stub_if_missing("imreg_dft", fake_imreg_dft) +_set_stub_if_missing("colour_demosaicing", fake_colour_demosaicing) +_set_stub_if_missing("photutils", fake_photutils) +_set_stub_if_missing("photutils.aperture", fake_photutils_aperture) +_set_stub_if_missing("photutils.detection", fake_photutils_detection) +_set_stub_if_missing("ldtk", fake_ldtk) +_set_stub_if_missing("ldtk.ldmodel", fake_ldtk_ldmodel) +_set_stub_if_missing("lmfit", fake_lmfit) +_set_stub_if_missing("pylightcurve", fake_pylightcurve) +_set_stub_if_missing("pylightcurve.models", fake_pylightcurve_models) +_set_stub_if_missing("pylightcurve.models.exoplanet_lc", fake_pylightcurve_exoplanet) +_set_stub_if_missing("pyvo", fake_pyvo) +_set_stub_if_missing("ultranest", fake_ultranest) +_set_stub_if_missing("barycorrpy", fake_barycorrpy) +_set_stub_if_missing("barycorrpy.utc_tdb", fake_utc_tdb) +sys.modules.setdefault("exotic.api.elca", fake_elca) +sys.modules.setdefault("exotic.api.ld", fake_ld) + +from exotic.exotic import ( # noqa: E402 + ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT, + INITIAL_RPRS_BOUND_LOWER_SCALE, + INITIAL_RPRS_BOUND_UPPER_SCALE, + RPRS_POSTERIOR_MAX_RETRIES_DEFAULT, + RPRS_SEARCH_BOUND_MAX, + RPRS_SEARCH_BOUND_MIN, + SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT, + TOI_TIC_ARS_POSTERIOR_MAX_RETRIES_DEFAULT, + TOI_TIC_ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT, + ars_initial_range_percentage_for_prior, + ars_posterior_retry_limit_for_prior, + ars_range_restriction_percentage_for_prior, + build_initial_ars_bounds, + build_fast_ultranest_lightcurve_series, + build_expected_transit_coverage_assessment, + evaluate_sparse_posterior_sample_support, + extend_sparse_posterior_live_points_if_needed, + final_residual_rejection_keep_mask, + fit_final_lightcurve_with_oot_baseline_detrending, + finalize_comparison_candidate_full_reduction, + refit_selected_fast_comparison_on_full_lightcurve, + configure_ars_range_restriction, + configure_prior_rprs_fallback_on_pinned_posterior, + configure_rprs_search_bound_max, + configure_rprs_range_restriction, + configured_prior_centered_bounds_for_key, + is_toi_or_tic_target, + should_use_legacy_psf_flux_mode, + should_run_final_fit_phase_residual_clip, + should_run_final_residual_rejection, + should_run_fast_ultranest_before_final_run, + should_restrict_ars_range, + should_restrict_rprs_range, + should_use_prior_rprs_when_posterior_pinned, + build_single_transit_duration_prior, + build_initial_rprs_bounds, + run_nested_lightcurve_fit_with_rprs_posterior_retry, +) + + +@pytest.fixture(autouse=True) +def _disable_prior_centered_range_restrictions(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_ENABLED", False) + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_PERCENTAGE", 10.0) + monkeypatch.setattr(exotic_module, "RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR", True) + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_ENABLED", False) + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_PERCENTAGE", 10.0) + + +def test_build_initial_rprs_bounds_allows_zero_depth_search_box(): + bounds = build_initial_rprs_bounds(0.1) + + assert bounds == pytest.approx([ + RPRS_SEARCH_BOUND_MIN, + INITIAL_RPRS_BOUND_UPPER_SCALE * 0.1, + ]) + assert INITIAL_RPRS_BOUND_LOWER_SCALE == pytest.approx(0.0) + + +def test_build_initial_rprs_bounds_restricts_to_prior_centered_window(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_ENABLED", True) + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_PERCENTAGE", 10.0) + + assert build_initial_rprs_bounds(0.1) == pytest.approx([0.09, 0.11]) + + +def test_build_initial_rprs_bounds_widens_prior_window_for_data_uncertainty(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_ENABLED", True) + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_PERCENTAGE", 10.0) + + assert build_initial_rprs_bounds(0.1, rprs_data_uncertainty=0.006) == pytest.approx([0.09, 0.11]) + assert build_initial_rprs_bounds(0.1, rprs_data_uncertainty=0.02) == pytest.approx([0.04, 0.16]) + + +def test_build_initial_ars_bounds_uses_larger_of_percentage_and_uncertainty_window(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_ENABLED", True) + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_PERCENTAGE", 10.0) + + assert build_initial_ars_bounds(15.0, None) == pytest.approx([11.25, 18.75]) + assert build_initial_ars_bounds(15.0, 0.1) == pytest.approx([13.5, 16.5]) + + +@pytest.mark.parametrize( + "target_name", + ["TOI-2969 b", "toi 2969.01", "TIC 123456789", "TIC-123456789"], +) +def test_toi_tic_target_detection_accepts_catalog_candidate_names(target_name): + assert is_toi_or_tic_target({"pName": target_name}) is True + + +@pytest.mark.parametrize("target_name", ["WASP-194 b", "HAT-P-32 b", "TOIL-1 b", None]) +def test_toi_tic_target_detection_rejects_established_or_unrelated_names(target_name): + assert is_toi_or_tic_target({"pName": target_name}) is False + + +def test_toi_tic_ars_policy_uses_thirty_percent_ceiling_and_more_retries(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_ENABLED", True) + monkeypatch.setattr( + exotic_module, + "ARS_RANGE_RESTRICTION_PERCENTAGE", + ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT, + ) + candidate_prior = { + "pName": "TOI-2969 b", + "sName": "TOI-2969", + "ars": 8.0, + "ars_unc": 0.4, + } + established_prior = { + "pName": "WASP-194 b", + "sName": "WASP-194", + "ars": 8.0, + "ars_unc": 0.4, + } + + assert ars_range_restriction_percentage_for_prior(candidate_prior) == pytest.approx( + TOI_TIC_ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT + ) + assert ars_initial_range_percentage_for_prior(candidate_prior) == pytest.approx(30.0) + assert configured_prior_centered_bounds_for_key("ars", candidate_prior) == pytest.approx([5.6, 10.4]) + assert configured_prior_centered_bounds_for_key("ars", established_prior) == pytest.approx([6.0, 10.0]) + assert build_initial_ars_bounds( + 8.0, + 0.4, + search_restriction_prior=candidate_prior, + ) == pytest.approx([5.6, 10.4]) + assert build_initial_ars_bounds( + 8.0, + 0.4, + search_restriction_prior=established_prior, + ) == pytest.approx([6.0, 10.0]) + high_uncertainty_candidate_prior = { + **candidate_prior, + "ars_unc": 0.8, + } + assert ars_initial_range_percentage_for_prior(high_uncertainty_candidate_prior) == pytest.approx(50.0) + assert build_initial_ars_bounds( + 8.0, + 0.8, + search_restriction_prior=high_uncertainty_candidate_prior, + ) == pytest.approx([4.0, 12.0]) + assert ars_posterior_retry_limit_for_prior(candidate_prior, 5) == ( + TOI_TIC_ARS_POSTERIOR_MAX_RETRIES_DEFAULT + ) + assert ars_posterior_retry_limit_for_prior(established_prior, 5) == 5 + assert ars_posterior_retry_limit_for_prior(candidate_prior, 2) == 2 + + +def test_toi_tic_ars_policy_preserves_explicit_nondefault_restriction(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_PERCENTAGE", 12.5) + + assert ars_range_restriction_percentage_for_prior( + {"pName": "TIC 123456789", "ars": 8.0} + ) == pytest.approx(12.5) + + +def test_build_initial_rprs_bounds_clamps_to_configured_search_ceiling(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "RPRS_SEARCH_BOUND_MAX", 0.5) + + assert build_initial_rprs_bounds(0.2) == pytest.approx([RPRS_SEARCH_BOUND_MIN, 0.5]) + assert build_initial_rprs_bounds(0.7) == pytest.approx([RPRS_SEARCH_BOUND_MIN, 0.5]) + + +def test_configure_rprs_search_bound_max_updates_retry_ceiling(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "RPRS_SEARCH_BOUND_MAX", 0.5) + + assert configure_rprs_search_bound_max("0.4") == pytest.approx(0.4) + assert exotic_module.RPRS_SEARCH_BOUND_MAX == pytest.approx(0.4) + + +def test_configure_prior_centered_range_restrictions_parse_values(): + import exotic.exotic as exotic_module + + assert should_restrict_rprs_range(None) is True + assert should_restrict_ars_range(None) is True + assert should_use_prior_rprs_when_posterior_pinned(None) is True + assert should_restrict_rprs_range("n") is False + assert should_restrict_ars_range(False) is False + assert should_use_prior_rprs_when_posterior_pinned("n") is False + + enabled, percentage = configure_rprs_range_restriction("y", "12.5%") + assert enabled is True + assert percentage == pytest.approx(12.5) + assert exotic_module.RPRS_RANGE_RESTRICTION_ENABLED is True + assert exotic_module.RPRS_RANGE_RESTRICTION_PERCENTAGE == pytest.approx(12.5) + + enabled, percentage = configure_ars_range_restriction("n", None) + assert enabled is False + assert percentage == pytest.approx(ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT) + assert exotic_module.ARS_RANGE_RESTRICTION_ENABLED is False + + assert configure_prior_rprs_fallback_on_pinned_posterior("n") is False + assert exotic_module.RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR is False + + +def test_rprs_posterior_retry_clamps_to_configured_search_ceiling(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "RPRS_SEARCH_BOUND_MAX", 0.5) + captured = {"calls": []} + diagnostics_sequence = [ + {"clipped": True, "edge": "upper", "mode": 0.49, "std": 0.10, "bounds": [0.29, 0.89]}, + {"clipped": True, "edge": "upper", "mode": 0.49, "std": 0.08, "bounds": [0.38, 0.78]}, + ] + + def make_fit(diagnostics): + fit = types.SimpleNamespace( + parameters={"tmid": 0.0, "rprs": diagnostics["mode"], "inc": 89.0, "a2": 0.0} + ) + fit.get_parameter_posterior_recenter_diagnostics = ( + lambda key: dict(diagnostics) if key == "rprs" else None + ) + return fit + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + call_index = len(captured["calls"]) + captured["calls"].append({ + "prior": dict(call_prior), + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + return make_fit(diagnostics_sequence[min(call_index, len(diagnostics_sequence) - 1)]) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.linspace(-0.03, 0.03, 7) + flux = np.ones(7, dtype=float) + fluxerr = np.full(7, 0.01, dtype=float) + airmass = np.ones(7, dtype=float) + prior = {"tmid": 0.0, "rprs": 0.4, "inc": 89.0, "a2": 0.0} + bounds = {"rprs": [0.0, 0.4], "tmid": [-0.01, 0.01], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]} + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + ) + + assert len(captured["calls"]) == 3 + assert captured["calls"][1]["bounds"]["rprs"][0] == pytest.approx(0.0) + assert captured["calls"][1]["bounds"]["rprs"][1] == pytest.approx(0.5) + assert "rprs" not in captured["calls"][2]["bounds"] + assert fit.rprs_posterior_refit_bounds[1] == pytest.approx(0.5) + assert fit.rprs_posterior_refit_applied is True + assert fit.rprs_prior_fallback_applied is True + assert fit.parameters["rprs"] == pytest.approx(0.4) + + +def test_expanded_prior_retry_passes_previous_fit_as_corrected_warmstart_source(monkeypatch): + import exotic.exotic as exotic_module + + calls = [] + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + ultranest_warmstart_source=None, + ): + call_index = len(calls) + fit = types.SimpleNamespace( + parameters={ + "tmid": 0.0, + "rprs": 0.19 if call_index == 0 else 0.21, + "inc": 89.0, + "a2": 0.0, + }, + ) + + def diagnostics(key): + if key != "rprs": + return None + if call_index == 0: + return { + "clipped": True, + "edge": "upper", + "mode": 0.19, + "std": 0.02, + "bounds": [0.15, 0.30], + } + return { + "clipped": False, + "edge": None, + "mode": 0.21, + "std": 0.02, + "bounds": list(call_bounds["rprs"]), + } + + fit.get_parameter_posterior_recenter_diagnostics = diagnostics + calls.append({ + "fit": fit, + "warmstart_source": ultranest_warmstart_source, + "bounds": {key: list(value) for key, value in call_bounds.items()}, + }) + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 7), + np.ones(7, dtype=float), + np.full(7, 0.01, dtype=float), + np.ones(7, dtype=float), + {"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0}, + { + "rprs": [0.0, 0.2], + "tmid": [-0.01, 0.01], + "inc": [84.0, 90.0], + "a2": [-3.0, 3.0], + }, + use_prior_rprs_when_posterior_pinned=False, + ) + + assert len(calls) == 2 + assert calls[0]["warmstart_source"] is None + assert calls[1]["warmstart_source"] is calls[0]["fit"] + assert calls[1]["bounds"]["rprs"] == pytest.approx([0.0, 0.30]) + assert fit.rprs_posterior_refit_applied is True + + +def test_rprs_posterior_retry_does_not_escape_configured_prior_range(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_ENABLED", True) + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_PERCENTAGE", 10.0) + monkeypatch.setattr(exotic_module, "RPRS_SEARCH_BOUND_MAX", 0.5) + captured = {"calls": []} + diagnostics = {"clipped": True, "edge": "upper", "mode": 0.109, "std": 0.04, "bounds": [0.09, 0.25]} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + **kwargs, + ): + captured["calls"].append({ + "prior": dict(call_prior), + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + fit = types.SimpleNamespace( + parameters={"tmid": 0.0, "rprs": diagnostics["mode"], "inc": 89.0, "a2": 0.0} + ) + fit.get_parameter_posterior_recenter_diagnostics = ( + lambda key: dict(diagnostics) if key == "rprs" else None + ) + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.linspace(-0.03, 0.03, 7) + flux = np.ones(7, dtype=float) + fluxerr = np.full(7, 0.01, dtype=float) + airmass = np.ones(7, dtype=float) + prior = {"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0} + bounds = {"rprs": [0.0, 0.3], "tmid": [-0.01, 0.01], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]} + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + ) + + assert len(captured["calls"]) == 2 + assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([0.09, 0.11]) + assert "rprs" not in captured["calls"][1]["bounds"] + assert fit.rprs_posterior_refit_applied is False + assert fit.rprs_prior_fallback_applied is True + assert fit.parameters["rprs"] == pytest.approx(0.1) + assert "prior fallback" in fit.rprs_prior_fallback_note + + +def test_rprs_restriction_uses_explicit_search_prior_instead_of_refined_prior(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_ENABLED", True) + monkeypatch.setattr(exotic_module, "RPRS_RANGE_RESTRICTION_PERCENTAGE", 10.0) + captured = {"bounds": None} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + **kwargs, + ): + captured["bounds"] = { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + } + fit = types.SimpleNamespace(parameters=dict(call_prior)) + fit.get_parameter_posterior_recenter_diagnostics = lambda key: { + "clipped": False, + "edge": None, + "mode": call_prior.get(key, np.nan), + "std": 0.01, + "bounds": captured["bounds"].get(key), + "reason": "posterior support is comfortably inside the sampled bounds.", + } if key == "rprs" else None + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.linspace(-0.03, 0.03, 7) + flux = np.ones(7, dtype=float) + fluxerr = np.full(7, 0.01, dtype=float) + airmass = np.ones(7, dtype=float) + + run_nested_lightcurve_fit_with_rprs_posterior_retry( + times, + flux, + fluxerr, + airmass, + {"tmid": 0.0, "rprs": 0.145, "inc": 89.0, "a2": 0.0}, + {"rprs": [0.0, 0.5], "tmid": [-0.01, 0.01], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]}, + search_restriction_prior={"rprs": 0.1}, + ) + + assert captured["bounds"]["rprs"] == pytest.approx([0.09, 0.11]) + + +def test_pinned_rprs_without_expansion_reruns_with_prior_value_and_data_error(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR", True) + + def fake_transit(call_times, call_prior): + call_times = np.asarray(call_times, dtype=float) + depth = float(call_prior["rprs"]) ** 2 + return 1.0 - depth * (np.abs(call_times) <= 0.01) + + monkeypatch.setattr(exotic_module, "transit", fake_transit) + + captured = {"calls": []} + pinned_diagnostics = { + "clipped": True, + "edge": "upper", + "mode": 0.11, + "std": 0.006, + "bounds": [0.09, 0.11], + } + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + fixed_parameter_errors=None, + **kwargs, + ): + captured["calls"].append({ + "prior": dict(call_prior), + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + "fixed_parameter_errors": dict(fixed_parameter_errors or {}), + }) + model = fake_transit(call_times, call_prior) + fit = types.SimpleNamespace( + time=np.asarray(call_times, dtype=float), + data=np.asarray(call_flux, dtype=float), + dataerr=np.asarray(call_fluxerr, dtype=float), + transit=np.asarray(model, dtype=float), + model=np.asarray(model, dtype=float), + residuals=np.asarray(call_flux, dtype=float) - np.asarray(model, dtype=float), + airmass_model=np.ones_like(model), + parameters=dict(call_prior), + errors=dict(fixed_parameter_errors or {}), + fixed_parameter_errors=dict(fixed_parameter_errors or {}), + ) + fit.errors.setdefault("tmid", 0.001) + fit.errors.setdefault("ars", 0.1) + fit.errors.setdefault("inc", 0.1) + + def diagnostics(key): + if key == "rprs" and "rprs" in call_bounds: + return dict(pinned_diagnostics) + return { + "clipped": False, + "edge": None, + "mode": call_prior.get(key, np.nan), + "std": 0.001, + "bounds": call_bounds.get(key), + "reason": "posterior support is comfortably inside the sampled bounds.", + } + + fit.get_parameter_posterior_recenter_diagnostics = diagnostics + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.linspace(-0.03, 0.03, 9) + prior = {"tmid": 0.0, "rprs": 0.1, "ars": 10.0, "inc": 89.0, "a2": 0.0} + flux = fake_transit(times, prior) + np.array([0.0, 0.004, -0.003, 0.002, -0.004, 0.003, -0.002, 0.004, 0.0]) + fluxerr = np.full(times.shape, 0.003) + airmass = np.ones(times.shape) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + times, + flux, + fluxerr, + airmass, + {"tmid": 0.0, "rprs": 0.11, "ars": 10.0, "inc": 89.0, "a2": 0.0}, + {"rprs": [0.09, 0.11], "tmid": [-0.01, 0.01], "ars": [8.0, 12.0], "inc": [84.0, 90.0]}, + max_rprs_retries=0, + max_ars_retries=0, + max_impact_parameter_retries=0, + search_restriction_prior={"rprs": 0.1, "ars": 10.0}, + ) + + assert len(captured["calls"]) == 2 + assert "rprs" in captured["calls"][0]["bounds"] + assert "rprs" not in captured["calls"][1]["bounds"] + assert captured["calls"][1]["prior"]["rprs"] == pytest.approx(0.1) + assert captured["calls"][1]["fixed_parameter_errors"]["rprs"] > 0 + assert fit.rprs_prior_fallback_applied is True + assert fit.parameters["rprs"] == pytest.approx(0.1) + assert fit.errors["rprs"] == pytest.approx(fit.empirical_transit_uncertainty["data_rprs_uncertainty"]) + assert fit.empirical_transit_uncertainty["combined_rprs_uncertainty"] == pytest.approx( + fit.empirical_transit_uncertainty["data_rprs_uncertainty"] + ) + assert "prior fallback" in fit.rprs_prior_fallback_note + + +def test_pinned_rprs_after_retry_cap_reruns_with_prior_value(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "RPRS_PRIOR_FALLBACK_ON_PINNED_POSTERIOR", True) + + def fake_transit(call_times, call_prior): + call_times = np.asarray(call_times, dtype=float) + depth = float(call_prior["rprs"]) ** 2 + return 1.0 - depth * (np.abs(call_times) <= 0.01) + + monkeypatch.setattr(exotic_module, "transit", fake_transit) + + captured = {"calls": []} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + fixed_parameter_errors=None, + **kwargs, + ): + call_index = len(captured["calls"]) + captured["calls"].append({ + "prior": dict(call_prior), + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + "fixed_parameter_errors": dict(fixed_parameter_errors or {}), + }) + model = fake_transit(call_times, call_prior) + fit = types.SimpleNamespace( + time=np.asarray(call_times, dtype=float), + data=np.asarray(call_flux, dtype=float), + dataerr=np.asarray(call_fluxerr, dtype=float), + transit=np.asarray(model, dtype=float), + model=np.asarray(model, dtype=float), + residuals=np.asarray(call_flux, dtype=float) - np.asarray(model, dtype=float), + airmass_model=np.ones_like(model), + parameters=dict(call_prior), + errors=dict(fixed_parameter_errors or {}), + fixed_parameter_errors=dict(fixed_parameter_errors or {}), + ) + fit.errors.setdefault("tmid", 0.001) + fit.errors.setdefault("ars", 0.1) + fit.errors.setdefault("inc", 0.1) + + def diagnostics(key): + if key == "rprs" and "rprs" in call_bounds: + return { + "clipped": True, + "edge": "upper", + "mode": call_bounds["rprs"][1], + "std": 0.006, + "bounds": [ + call_bounds["rprs"][0] + 0.01, + call_bounds["rprs"][1] + 0.01, + ], + "reason": "posterior peaks against the upper search bound.", + } + return { + "clipped": False, + "edge": None, + "mode": call_prior.get(key, np.nan), + "std": 0.001, + "bounds": call_bounds.get(key), + "reason": "posterior support is comfortably inside the sampled bounds.", + } + + fit.get_parameter_posterior_recenter_diagnostics = diagnostics + fit.call_index = call_index + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.linspace(-0.03, 0.03, 9) + prior = {"tmid": 0.0, "rprs": 0.1, "ars": 10.0, "inc": 89.0, "a2": 0.0} + flux = fake_transit(times, prior) + np.array([0.0, 0.004, -0.003, 0.002, -0.004, 0.003, -0.002, 0.004, 0.0]) + fluxerr = np.full(times.shape, 0.003) + airmass = np.ones(times.shape) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + times, + flux, + fluxerr, + airmass, + {"tmid": 0.0, "rprs": 0.11, "ars": 10.0, "inc": 89.0, "a2": 0.0}, + {"rprs": [0.09, 0.11], "tmid": [-0.01, 0.01], "ars": [8.0, 12.0], "inc": [84.0, 90.0]}, + max_rprs_retries=1, + max_ars_retries=0, + max_impact_parameter_retries=0, + search_restriction_prior={"rprs": 0.1, "ars": 10.0}, + ) + + assert len(captured["calls"]) == 3 + assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.06, 0.16]) + assert "rprs" not in captured["calls"][2]["bounds"] + assert captured["calls"][2]["prior"]["rprs"] == pytest.approx(0.1) + assert fit.rprs_prior_fallback_applied is True + assert fit.parameters["rprs"] == pytest.approx(0.1) + assert "prior fallback" in fit.rprs_prior_fallback_note + assert "then applied the Rp/R* prior fallback" in fit.rprs_posterior_refit_note + + +def test_fast_ultranest_option_defaults_enabled_and_parses_false_values(): + assert should_run_fast_ultranest_before_final_run(None) is True + assert should_run_fast_ultranest_before_final_run("n") is False + assert should_run_fast_ultranest_before_final_run(False) is False + + +def test_final_residual_rejection_option_defaults_enabled_and_parses_false_values(): + assert should_run_final_residual_rejection(None) is True + assert should_run_final_residual_rejection("n") is False + assert should_run_final_residual_rejection(False) is False + + +def test_final_fit_phase_residual_clip_option_defaults_enabled_and_parses_false_values(): + assert should_run_final_fit_phase_residual_clip(None) is True + assert should_run_final_fit_phase_residual_clip("n") is False + assert should_run_final_fit_phase_residual_clip(False) is False + + +def test_legacy_psf_flux_mode_option_defaults_modern_and_parses_true_values(): + assert should_use_legacy_psf_flux_mode(None) is False + assert should_use_legacy_psf_flux_mode("legacy") is True + assert should_use_legacy_psf_flux_mode("y") is True + assert should_use_legacy_psf_flux_mode(False) is False + + +def test_final_residual_rejection_keep_mask_flags_large_residual_outlier(): + fit = types.SimpleNamespace( + data=np.ones(8, dtype=float), + residuals=np.array([0.0, 0.001, -0.001, 0.0, 0.001, -0.001, 0.0, 0.20], dtype=float), + ) + + keep_mask, summary = final_residual_rejection_keep_mask(fit, sigma=2.0, min_required_points=5) + + assert keep_mask.tolist() == [True, True, True, True, True, True, True, False] + assert summary["applied"] is True + assert summary["rejected_point_count"] == 1 + assert summary["kept_point_count"] == 7 + + +def test_final_residual_rejection_keep_mask_iterates_until_clean(): + fit = types.SimpleNamespace( + data=np.ones(10, dtype=float), + residuals=np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.05, 0.20], dtype=float), + ) + + keep_mask, summary = final_residual_rejection_keep_mask(fit, sigma=2.0, min_required_points=5) + + assert keep_mask.tolist() == [True, True, True, True, True, True, True, True, False, False] + assert summary["applied"] is True + assert summary["rejected_point_count"] == 2 + assert summary["clip_iteration_count"] == 2 + + +def test_fast_ultranest_binning_reduces_large_light_curve_to_twenty_points(): + times = np.linspace(0.0, 1.0, 80) + flux = 1.0 + 0.01 * np.sin(np.linspace(0.0, 2.0 * np.pi, 80)) + unc = np.full(80, 0.01) + airmass = np.linspace(1.0, 1.5, 80) + + result = build_fast_ultranest_lightcurve_series(times, flux, unc, airmass) + + assert result["applied"] is True + assert result["original_point_count"] == 80 + assert result["binned_point_count"] <= 20 + assert result["time"].shape == result["flux"].shape == result["unc"].shape == result["airmass"].shape + + +def test_fast_ultranest_binning_skips_short_light_curve(): + times = np.linspace(0.0, 1.0, 60) + result = build_fast_ultranest_lightcurve_series( + times, + np.ones(60), + np.full(60, 0.01), + np.linspace(1.0, 1.2, 60), + ) + + assert result["applied"] is False + assert result["binned_point_count"] == 60 + + +def test_expected_transit_coverage_assessment_flags_ingress_only_as_very_low(): + prior = {"tmid": 10.0, "per": 2.0, "rprs": 0.1, "ars": 12.0, "inc": 89.0, "ecc": 0.0, "omega": 0.0} + duration_prior = {"applied": True, "expected_duration": 0.1} + times = np.linspace(9.90, 9.955, 12) + + assessment = build_expected_transit_coverage_assessment( + times, + prior, + flux_values=np.ones(times.shape[0]), + flux_errors=np.full(times.shape[0], 0.001), + duration_prior=duration_prior, + ) + + assert assessment["valid"] is True + assert assessment["observed_segment"] == "pre-ingress baseline plus ingress" + assert assessment["transit_fraction_observed"] == pytest.approx(0.05) + assert assessment["success_label"] == "very low" + assert assessment["expected_successful"] is False + + +def test_final_fit_logs_partial_coverage_before_first_ultranest_call(monkeypatch): + import exotic.exotic as exotic_module + + events = [] + + def fake_log_info(message, warn=False, error=False): + events.append(("log", str(message), warn)) + return True + + def fake_run_nested(times, flux_values, flux_errors, airmass, prior, bounds, **kwargs): + events.append(("run_nested", "", False)) + local_times = np.asarray(times, dtype=float) + model = np.ones(local_times.shape[0], dtype=float) + model[-1:] -= 0.01 + fit = types.SimpleNamespace( + time=local_times, + data=np.asarray(flux_values, dtype=float), + dataerr=np.asarray(flux_errors, dtype=float), + model=model, + residuals=np.zeros(local_times.shape[0], dtype=float), + airmass=np.asarray(airmass, dtype=float), + parameters={"rprs": prior["rprs"], "tmid": prior["tmid"], "inc": prior["inc"], "a2": 0.0, "per": prior["per"]}, + errors={"rprs": 0.01, "tmid": 0.001, "inc": 0.1, "a2": 0.01}, + bounds=dict(bounds), + duration_expected=0.1, + duration_measured=0.1, + ) + fit.get_parameter_posterior_recenter_diagnostics = ( + lambda key: {"clipped": False, "edge": None, "mode": fit.parameters.get(key, np.nan), "std": 0.01} + ) + return fit + + monkeypatch.setattr(exotic_module, "log_info", fake_log_info) + monkeypatch.setattr(exotic_module, "run_nested_lightcurve_fit_with_rprs_posterior_retry", fake_run_nested) + monkeypatch.setattr(exotic_module, "apply_plot_time_range", lambda fit, plot_time_range: fit) + monkeypatch.setattr( + exotic_module, + "build_final_fit_prefit_refinement_plan", + lambda times, flux_values, flux_errors, airmass, prior, bounds, fit, **kwargs: { + "applied": False, + "note": "not needed", + "times": np.asarray(times, dtype=float), + "flux": np.asarray(flux_values, dtype=float), + "unc": np.asarray(flux_errors, dtype=float), + "airmass": np.asarray(airmass, dtype=float), + "jd_times": None, + "prior": dict(prior), + "bounds": dict(bounds), + "duration": 0.1, + "original_point_count": len(times), + "refined_point_count": len(times), + "trimmed_pre_points": 0, + "trimmed_post_points": 0, + "original_tmid_bounds": bounds["tmid"], + "refined_tmid_bounds": bounds["tmid"], + }, + ) + + times = np.linspace(9.90, 9.955, 12) + fit, _, _ = fit_final_lightcurve_with_oot_baseline_detrending( + times, + np.ones(times.shape[0], dtype=float), + np.full(times.shape[0], 0.001, dtype=float), + np.linspace(1.0, 1.1, times.shape[0]), + {"rprs": 0.1, "tmid": 10.0, "inc": 89.0, "a2": 0.0, "per": 2.0, "ars": 12.0, "ecc": 0.0, "omega": 0.0}, + {"rprs": [0.0, 0.5], "tmid": [9.95, 10.05], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]}, + detrend_on_outoftransit_baseline=False, + duration_prior={"applied": True, "expected_duration": 0.1}, + ) + + coverage_index = next(i for i, event in enumerate(events) if "Pre-UltraNest transit coverage assessment" in event[1]) + nested_index = next(i for i, event in enumerate(events) if event[0] == "run_nested") + assert coverage_index < nested_index + assert any("Estimated fit success: VERY LOW" in event[1] and event[2] for event in events) + assert fit.pre_ultranest_transit_coverage_status == "very low" + + +def test_finalize_comparison_candidate_runs_pre_final_ultranest_on_binned_series(monkeypatch): + import exotic.exotic as exotic_module + + captured = {} + + def fake_fit_final( + times, + flux_values, + flux_errors, + airmass, + prior, + bounds, + jd_times=None, + **kwargs, + ): + captured["point_count"] = len(times) + captured["bounds"] = dict(bounds) + captured["fix_baseline_terms_for_final"] = kwargs.get("fix_baseline_terms_for_final") + fit = types.SimpleNamespace( + time=np.asarray(times, dtype=float), + data=np.asarray(flux_values, dtype=float), + dataerr=np.asarray(flux_errors, dtype=float), + airmass=np.asarray(airmass, dtype=float), + parameters={ + **dict(prior), + "tmid": 0.5, + "rprs": 0.1, + "ars": 10.0, + "inc": 89.0, + "a0": 1.0, + "a1": 1.0, + "a2": 0.02, + }, + errors={"tmid": 0.001, "rprs": 0.001, "ars": 0.1, "inc": 0.1, "a0": 0.01, "a2": 0.01}, + transit=np.ones(len(times), dtype=float), + residuals=np.zeros(len(times), dtype=float), + duration_measured=0.04, + duration_expected=0.04, + ) + return fit, np.asarray(flux_values, dtype=float), np.asarray(flux_errors, dtype=float) + + monkeypatch.setattr(exotic_module, "fit_final_lightcurve_with_oot_baseline_detrending", fake_fit_final) + + times = np.linspace(0.0, 1.0, 80) + target_flux = 100.0 * (1.0 + 0.002 * np.sin(np.linspace(0.0, 2.0 * np.pi, 80))) + result = finalize_comparison_candidate_full_reduction( + times, + target_flux, + np.full(80, 100.0), + np.linspace(1.0, 1.3, 80), + ld=[0.1, 0.1, 0.1, 0.1], + p_dict={ + "pName": "Test b", + "midT": 0.5, + "midTUnc": 0.001, + "pPer": 1.0, + "pPerUnc": 0.001, + "rprs": 0.1, + "aRs": 10.0, + "aRsUnc": 0.1, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + }, + jd_times=2460000.0 + times, + run_fast_ultranest_before_final_run=True, + ) + + assert result["applied"] is True + assert captured["point_count"] <= 20 + assert captured["fix_baseline_terms_for_final"] is False + assert "a0" in captured["bounds"] + assert "a2" in captured["bounds"] + assert len(result["good_times"]) == result["fast_ultranest_binning"]["original_point_count"] + assert len(result["good_times"]) > 60 + assert result["fast_ultranest_binning"]["applied"] is True + assert result["fit"].fast_ultranest_binning_applied is True + + +def test_selected_fast_candidate_final_refit_uses_full_series_and_fixed_baseline(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setenv("EXOTIC_ULTRANEST_MIN_NUM_LIVE_POINTS", "200") + monkeypatch.setenv("EXOTIC_SPARSE_POSTERIOR_LIVE_POINT_RETRY", "1") + captured = {} + + def fake_run_nested( + times, + flux_values, + flux_errors, + airmass, + prior, + bounds, + jd_times=None, + **kwargs, + ): + captured["point_count"] = len(times) + captured["prior"] = dict(prior) + captured["bounds"] = dict(bounds) + captured["fixed_parameter_errors"] = dict(kwargs.get("fixed_parameter_errors", {})) + captured["fixed_flux_baseline"] = kwargs.get("fixed_flux_baseline") + captured["ultranest_min_num_live_points"] = kwargs.get("ultranest_min_num_live_points") + captured["max_rprs_retries"] = kwargs.get("max_rprs_retries") + captured["max_ars_retries"] = kwargs.get("max_ars_retries") + captured["max_impact_parameter_retries"] = kwargs.get("max_impact_parameter_retries") + fit = types.SimpleNamespace( + time=np.asarray(times, dtype=float), + data=np.asarray(flux_values, dtype=float), + dataerr=np.asarray(flux_errors, dtype=float), + airmass=np.asarray(airmass, dtype=float), + parameters=dict(prior), + errors=dict(kwargs.get("fixed_parameter_errors", {})), + residuals=np.zeros(len(times), dtype=float), + transit=np.ones(len(times), dtype=float), + duration_measured=0.04, + duration_expected=0.04, + transit_qc={"status": "pass", "summary": "ok"}, + transit_qc_status="pass", + ) + fit.get_parameter_posterior_samples = lambda key: np.linspace(0.0, 1.0, 1500) + return fit + + monkeypatch.setattr(exotic_module, "run_nested_lightcurve_fit_with_rprs_posterior_retry", fake_run_nested) + + previous_fit = types.SimpleNamespace( + fast_ultranest_binning_applied=True, + parameters={ + "rprs": 0.1, + "ars": 10.0, + "per": 1.0, + "tmid": 0.5, + "inc": 89.0, + "u0": 0.1, + "u1": 0.1, + "u2": 0.1, + "u3": 0.1, + "ecc": 0.0, + "omega": 0.0, + "a0": 1.03, + "a1": 1.03, + "a2": 0.12, + }, + errors={"a0": 0.02, "a1": 0.02, "a2": 0.03, "rprs": 0.001, "tmid": 0.001, "ars": 0.1}, + bounds={ + "rprs": [0.05, 0.15], + "tmid": [0.49, 0.51], + "ars": [9.0, 11.0], + "inc": [85.0, 90.0], + "a0": [0.95, 1.05], + "a2": [-3.0, 3.0], + }, + ) + times = np.linspace(0.0, 1.0, 80) + selected_result = { + "fit": previous_fit, + "good_times": times, + "good_flux": np.ones(80), + "good_unc": np.full(80, 0.01), + "good_airmass": np.linspace(1.0, 1.3, 80), + "good_jd_times": 2460000.0 + times, + } + + returned = refit_selected_fast_comparison_on_full_lightcurve( + selected_result, + { + "midT": 0.5, + "midTUnc": 0.001, + "pPer": 1.0, + "rprs": 0.1, + "aRs": 10.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + }, + detrend_on_outoftransit_baseline=False, + ) + + assert returned is not None + assert captured["point_count"] == 80 + assert captured["fixed_flux_baseline"] is True + assert captured["ultranest_min_num_live_points"] == 1200 + assert captured["max_rprs_retries"] == 0 + assert captured["max_ars_retries"] == 0 + assert captured["max_impact_parameter_retries"] == 0 + assert captured["prior"]["a0"] == pytest.approx(1.03) + assert captured["prior"]["a2"] == pytest.approx(0.12) + assert captured["fixed_parameter_errors"]["a0"] == pytest.approx(0.02) + assert captured["fixed_parameter_errors"]["a2"] == pytest.approx(0.03) + assert "a0" not in captured["bounds"] + assert "a2" not in captured["bounds"] + + +def test_selected_fast_candidate_final_refit_estimates_missing_fixed_a2_error(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "selected_final_live_point_target", lambda *args, **kwargs: (200, None)) + captured = {} + + def fake_run_nested( + times, + flux_values, + flux_errors, + airmass, + prior, + bounds, + jd_times=None, + **kwargs, + ): + captured["fixed_parameter_errors"] = dict(kwargs.get("fixed_parameter_errors", {})) + fit = types.SimpleNamespace( + time=np.asarray(times, dtype=float), + data=np.asarray(flux_values, dtype=float), + dataerr=np.asarray(flux_errors, dtype=float), + airmass=np.asarray(airmass, dtype=float), + parameters=dict(prior), + errors=dict(kwargs.get("fixed_parameter_errors", {})), + residuals=np.zeros(len(times), dtype=float), + transit=np.ones(len(times), dtype=float), + duration_measured=0.04, + duration_expected=0.04, + transit_qc={"status": "pass", "summary": "ok"}, + transit_qc_status="pass", + ) + fit.get_parameter_posterior_samples = lambda key: np.linspace(0.0, 1.0, 1500) + return fit + + monkeypatch.setattr(exotic_module, "run_nested_lightcurve_fit_with_rprs_posterior_retry", fake_run_nested) + + previous_fit = types.SimpleNamespace( + fast_ultranest_binning_applied=True, + parameters={ + "rprs": 0.1, + "ars": 10.0, + "per": 1.0, + "tmid": 0.5, + "inc": 89.0, + "u0": 0.1, + "u1": 0.1, + "u2": 0.1, + "u3": 0.1, + "ecc": 0.0, + "omega": 0.0, + "a0": 1.03, + "a1": 1.03, + "a2": 0.12, + }, + errors={"a0": 0.02, "a1": 0.02, "rprs": 0.001, "tmid": 0.001, "ars": 0.1}, + data=np.ones(40, dtype=float), + dataerr=np.full(40, 0.01, dtype=float), + airmass=np.linspace(1.0, 1.4, 40), + bounds={ + "rprs": [0.05, 0.15], + "tmid": [0.49, 0.51], + "ars": [9.0, 11.0], + "inc": [85.0, 90.0], + "a0": [0.95, 1.05], + "a2": [-3.0, 3.0], + }, + ) + times = np.linspace(0.0, 1.0, 80) + selected_result = { + "fit": previous_fit, + "good_times": times, + "good_flux": np.ones(80), + "good_unc": np.full(80, 0.01), + "good_airmass": np.linspace(1.0, 1.3, 80), + "good_jd_times": 2460000.0 + times, + } + + returned = refit_selected_fast_comparison_on_full_lightcurve( + selected_result, + { + "midT": 0.5, + "midTUnc": 0.001, + "pPer": 1.0, + "rprs": 0.1, + "aRs": 10.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + }, + detrend_on_outoftransit_baseline=False, + ) + + assert returned is not None + assert captured["fixed_parameter_errors"]["a0"] == pytest.approx(0.02) + assert captured["fixed_parameter_errors"]["a2"] == pytest.approx(0.025) + + +def test_selected_fast_candidate_final_refit_reruns_after_residual_rejection(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "selected_final_live_point_target", lambda *args, **kwargs: (200, None)) + calls = [] + + def fake_run_nested( + times, + flux_values, + flux_errors, + airmass, + prior, + bounds, + jd_times=None, + **kwargs, + ): + call_index = len(calls) + calls.append({ + "times": np.asarray(times, dtype=float), + "flux": np.asarray(flux_values, dtype=float), + "source_count": len(times), + }) + residuals = np.zeros(len(times), dtype=float) + if call_index == 0: + residuals[-1] = 1.0 + elif call_index == 1: + residuals[0] = 1.0 + fit = types.SimpleNamespace( + time=np.asarray(times, dtype=float), + data=np.asarray(flux_values, dtype=float), + dataerr=np.asarray(flux_errors, dtype=float), + airmass=np.asarray(airmass, dtype=float), + parameters={**dict(prior), "rprs": 0.1, "tmid": 0.5, "ars": 10.0, "inc": 89.0}, + errors={"rprs": 0.001, "tmid": 0.001, "ars": 0.1, "inc": 0.1}, + residuals=residuals, + phase=np.linspace(-0.05, 0.05, len(times)), + detrended=np.asarray(flux_values, dtype=float), + transit=np.ones(len(times), dtype=float), + duration_measured=0.04, + duration_expected=0.04, + transit_qc={"status": "pass", "summary": "ok"}, + transit_qc_status="pass", + ) + return fit + + monkeypatch.setattr(exotic_module, "run_nested_lightcurve_fit_with_rprs_posterior_retry", fake_run_nested) + + previous_fit = types.SimpleNamespace( + fast_ultranest_binning_applied=True, + frame_filter_diagnostics=[], + parameters={ + "rprs": 0.1, + "ars": 10.0, + "per": 1.0, + "tmid": 0.5, + "inc": 89.0, + "u0": 0.1, + "u1": 0.1, + "u2": 0.1, + "u3": 0.1, + "ecc": 0.0, + "omega": 0.0, + "a0": 1.0, + "a1": 1.0, + "a2": 0.0, + }, + errors={"a0": 0.0, "a1": 0.0, "a2": 0.0, "rprs": 0.001, "tmid": 0.001, "ars": 0.1}, + bounds={ + "rprs": [0.05, 0.15], + "tmid": [0.49, 0.51], + "ars": [9.0, 11.0], + "inc": [85.0, 90.0], + }, + ) + times = np.linspace(0.0, 1.0, 80) + selected_result = { + "fit": previous_fit, + "good_times": times, + "good_flux": np.ones(80), + "good_unc": np.full(80, 0.01), + "good_airmass": np.linspace(1.0, 1.3, 80), + "good_jd_times": 2460000.0 + times, + "good_target_flux": np.linspace(1000.0, 1080.0, 80), + "good_comp_flux": np.linspace(500.0, 540.0, 80), + "source_indices": np.arange(80), + } + + returned, fit_flux, fit_unc = refit_selected_fast_comparison_on_full_lightcurve( + selected_result, + { + "midT": 0.5, + "midTUnc": 0.001, + "pPer": 1.0, + "rprs": 0.1, + "aRs": 10.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + }, + detrend_on_outoftransit_baseline=False, + ) + + assert returned is not None + assert [call["source_count"] for call in calls] == [80, 79, 78] + assert len(fit_flux) == 78 + assert len(fit_unc) == 78 + assert selected_result["good_times"].shape == (78,) + assert selected_result["tflux_fit"].shape == (78,) + assert selected_result["cflux_fit"].shape == (78,) + assert selected_result["source_indices"][0] == 1 + assert selected_result["source_indices"][-1] == 78 + assert returned.final_residual_rejection_applied is True + assert returned.final_residual_rejection_rejected_count == 2 + assert returned.final_residual_rejection["refit_iteration_count"] == 2 + assert returned.final_residual_rejection["rejected_source_indices"] == [79, 0] + assert [item["stage"] for item in returned.frame_filter_diagnostics[-2:]] == [ + "Final residual rejection refit 1", + "Final residual rejection refit 2", + ] + + +def test_selected_fast_candidate_final_refit_resets_fixed_baseline_after_linear_detrend(monkeypatch): + import exotic.exotic as exotic_module + + captured = {} + + def fake_run_nested( + times, + flux_values, + flux_errors, + airmass, + prior, + bounds, + jd_times=None, + **kwargs, + ): + captured["flux"] = np.asarray(flux_values, dtype=float) + captured["prior"] = dict(prior) + captured["bounds"] = dict(bounds) + captured["fixed_parameter_errors"] = dict(kwargs.get("fixed_parameter_errors", {})) + captured["fixed_flux_baseline"] = kwargs.get("fixed_flux_baseline") + fit = types.SimpleNamespace( + time=np.asarray(times, dtype=float), + data=np.asarray(flux_values, dtype=float), + dataerr=np.asarray(flux_errors, dtype=float), + airmass=np.asarray(airmass, dtype=float), + parameters=dict(prior), + errors=dict(kwargs.get("fixed_parameter_errors", {})), + residuals=np.zeros(len(times), dtype=float), + transit=np.ones(len(times), dtype=float), + duration_measured=0.2, + duration_expected=0.2, + transit_qc={"status": "pass", "summary": "ok"}, + transit_qc_status="pass", + ) + return fit + + monkeypatch.setattr(exotic_module, "run_nested_lightcurve_fit_with_rprs_posterior_retry", fake_run_nested) + monkeypatch.setattr(exotic_module, "selected_final_live_point_target", lambda *args, **kwargs: (200, None)) + + times = np.array([-2.0, -1.0, -0.25, 0.0, 0.25, 1.0, 2.0]) + transit_profile = np.array([1.0, 1.0, 1.0, 0.99, 1.0, 1.0, 1.0]) + baseline = 1.03 + 0.02 * times + previous_fit = types.SimpleNamespace( + fast_ultranest_binning_applied=True, + transit=transit_profile, + parameters={ + "rprs": 0.1, + "ars": 10.0, + "per": 1.0, + "tmid": 0.0, + "inc": 89.0, + "u0": 0.1, + "u1": 0.1, + "u2": 0.1, + "u3": 0.1, + "ecc": 0.0, + "omega": 0.0, + "a0": 1.03, + "a1": 1.03, + "a2": 0.12, + }, + errors={"a0": 0.02, "a1": 0.02, "a2": 0.03, "rprs": 0.001, "tmid": 0.001, "ars": 0.1}, + bounds={ + "rprs": [0.05, 0.15], + "tmid": [-0.1, 0.1], + "ars": [9.0, 11.0], + "inc": [85.0, 90.0], + "a0": [0.95, 1.05], + "a2": [-3.0, 3.0], + }, + ) + selected_result = { + "fit": previous_fit, + "good_times": times, + "good_flux": baseline * transit_profile, + "good_unc": np.full(times.shape, 0.01), + "good_airmass": np.linspace(1.0, 1.3, times.size), + "good_jd_times": 2460000.0 + times, + "fast_fit_bounds": previous_fit.bounds, + } + + returned, fit_flux, _ = refit_selected_fast_comparison_on_full_lightcurve( + selected_result, + { + "midT": 0.0, + "midTUnc": 0.001, + "pPer": 1.0, + "rprs": 0.1, + "aRs": 10.0, + "inc": 89.0, + "ecc": 0.0, + "omega": 0.0, + }, + detrend_on_outoftransit_baseline=True, + oot_baseline_min_points_per_side=2, + ) + + assert returned is not None + assert captured["fixed_flux_baseline"] is True + assert captured["prior"]["a0"] == pytest.approx(1.0) + assert captured["prior"]["a1"] == pytest.approx(1.0) + assert captured["prior"]["a2"] == pytest.approx(0.0) + assert captured["fixed_parameter_errors"]["a0"] == pytest.approx(0.02) + assert captured["fixed_parameter_errors"]["a2"] == pytest.approx(0.03) + assert np.allclose(captured["flux"][[0, 1, 2, 4, 5, 6]], 1.0, atol=1e-8) + assert captured["flux"][3] == pytest.approx(0.99, abs=1e-8) + assert np.allclose(fit_flux, captured["flux"]) + assert returned.oot_baseline_parameter_fit_applied is False + assert "instead of reusing" in returned.oot_baseline_parameter_fit_note + assert returned.pre_detrending_baseline_source.startswith("selected fast UltraNest fit") + assert returned.pre_detrending_baseline_scale_parameter == "a1" + assert returned.pre_detrending_baseline_scale_value == pytest.approx(1.03) + assert returned.pre_detrending_baseline_scale_error == pytest.approx(0.02) + assert returned.pre_detrending_baseline_a2_value == pytest.approx(0.12) + assert returned.pre_detrending_baseline_a2_error == pytest.approx(0.03) + + +def test_rprs_posterior_retry_walks_bounds_until_retry_cap(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + diagnostics_sequence = [ + {"clipped": True, "edge": "upper", "mode": 0.158, "std": 0.006, "bounds": [0.128, 0.188]}, + {"clipped": True, "edge": "upper", "mode": 0.182, "std": 0.005, "bounds": [0.157, 0.207]}, + {"clipped": True, "edge": "upper", "mode": 0.194, "std": 0.004, "bounds": [0.174, 0.214]}, + {"clipped": True, "edge": "upper", "mode": 0.201, "std": 0.003, "bounds": [0.186, 0.216]}, + {"clipped": True, "edge": "upper", "mode": 0.206, "std": 0.003, "bounds": [0.191, 0.221]}, + {"clipped": True, "edge": "upper", "mode": 0.210, "std": 0.003, "bounds": [0.195, 0.225]}, + ] + + def make_fit(diagnostics): + fit = types.SimpleNamespace( + parameters={ + "rprs": diagnostics["mode"], + "tmid": 0.0, + "inc": 89.0, + "a2": 0.0, + } + ) + + def get_parameter_posterior_recenter_diagnostics(key): + assert key == "rprs" + return dict(diagnostics) + + fit.get_parameter_posterior_recenter_diagnostics = get_parameter_posterior_recenter_diagnostics + return fit + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + call_index = len(captured["calls"]) + captured["calls"].append({ + "prior": dict(call_prior), + "duration_prior": duration_prior, + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + return make_fit(diagnostics_sequence[min(call_index, len(diagnostics_sequence) - 1)]) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.linspace(-0.03, 0.03, 7) + flux = np.ones(7, dtype=float) + fluxerr = np.full(7, 0.01, dtype=float) + airmass = np.ones(7, dtype=float) + prior = {"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0} + bounds = {"rprs": [0.0, 0.125], "tmid": [-0.01, 0.01], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]} + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + ) + + assert RPRS_POSTERIOR_MAX_RETRIES_DEFAULT == 5 + assert len(captured["calls"]) == 7 + assert "rprs" not in captured["calls"][-1]["bounds"] + np.testing.assert_allclose( + np.asarray([call["bounds"]["rprs"] for call in captured["calls"][:-1]], dtype=float), + np.asarray([ + [0.0, 0.125], + [0.0, 0.208], + [0.0, 0.232], + [0.0, 0.244], + [0.0, 0.251], + [0.0, 0.256], + ], dtype=float), + ) + assert fit.rprs_posterior_refit_applied is True + assert fit.rprs_posterior_refit_count == 5 + assert fit.rprs_posterior_refit_edge == "upper" + assert fit.rprs_posterior_refit_bounds == pytest.approx([0.0, 0.256]) + assert fit.rprs_prior_fallback_applied is True + assert fit.parameters["rprs"] == pytest.approx(0.1) + assert "prior fallback" in fit.rprs_prior_fallback_note + + +def test_rprs_posterior_retry_expands_bounds_without_hitting_the_old_0p3_cap(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + diagnostics_sequence = [ + {"clipped": True, "edge": "upper", "mode": 0.275, "std": 0.020, "bounds": [0.175, 0.375]}, + {"clipped": False, "edge": None, "mode": 0.278, "std": 0.012, "bounds": [0.175, RPRS_SEARCH_BOUND_MAX]}, + ] + + def make_fit(diagnostics): + fit = types.SimpleNamespace( + parameters={ + "rprs": diagnostics["mode"], + "tmid": 0.0, + "inc": 89.0, + "a2": 0.0, + } + ) + + def get_parameter_posterior_recenter_diagnostics(key): + assert key == "rprs" + return dict(diagnostics) + + fit.get_parameter_posterior_recenter_diagnostics = get_parameter_posterior_recenter_diagnostics + return fit + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + call_index = len(captured["calls"]) + captured["calls"].append({ + "prior": dict(call_prior), + "duration_prior": duration_prior, + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + return make_fit(diagnostics_sequence[call_index]) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.linspace(-0.03, 0.03, 7) + flux = np.ones(7, dtype=float) + fluxerr = np.full(7, 0.01, dtype=float) + airmass = np.ones(7, dtype=float) + prior = {"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0} + bounds = {"rprs": [0.0, 0.25], "tmid": [-0.01, 0.01], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]} + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + ) + + assert len(captured["calls"]) == 2 + assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([0.0, 0.25]) + assert captured["calls"][1]["prior"]["rprs"] == pytest.approx(0.275) + assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.0, 0.375]) + assert fit.rprs_posterior_refit_applied is True + assert fit.rprs_posterior_refit_count == 1 + assert fit.rprs_posterior_refit_bounds == pytest.approx([0.0, 0.375]) + + +def test_rprs_posterior_retry_can_continue_above_the_old_maximum_exoplanet_range(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + diagnostics = {"clipped": True, "edge": "upper", "mode": 0.275, "std": 0.020, "bounds": [0.175, 0.375]} + + def make_fit(): + fit = types.SimpleNamespace( + parameters={ + "rprs": diagnostics["mode"], + "tmid": 0.0, + "inc": 89.0, + "a2": 0.0, + } + ) + + def get_parameter_posterior_recenter_diagnostics(key): + assert key == "rprs" + return dict(diagnostics) + + fit.get_parameter_posterior_recenter_diagnostics = get_parameter_posterior_recenter_diagnostics + return fit + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + captured["calls"].append({ + "prior": dict(call_prior), + "duration_prior": duration_prior, + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + return make_fit() + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + times = np.linspace(-0.03, 0.03, 7) + flux = np.ones(7, dtype=float) + fluxerr = np.full(7, 0.01, dtype=float) + airmass = np.ones(7, dtype=float) + prior = {"tmid": 0.0, "rprs": 0.35, "inc": 89.0, "a2": 0.0} + bounds = {"rprs": [RPRS_SEARCH_BOUND_MIN, 0.35], "tmid": [-0.01, 0.01], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]} + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + times, + flux, + fluxerr, + airmass, + prior, + bounds, + ) + + assert len(captured["calls"]) == 3 + assert captured["calls"][0]["prior"]["rprs"] == pytest.approx(0.35) + assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([RPRS_SEARCH_BOUND_MIN, 0.35]) + assert captured["calls"][1]["prior"]["rprs"] == pytest.approx(0.275) + assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.0, 0.375]) + assert "rprs" not in captured["calls"][2]["bounds"] + assert fit.rprs_posterior_refit_applied is True + assert fit.rprs_posterior_refit_count == 1 + assert fit.rprs_prior_fallback_applied is True + + +def test_rprs_posterior_retry_expands_lower_edge_down_to_zero(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + diagnostics_sequence = [ + {"clipped": True, "edge": "lower", "mode": 0.030, "std": 0.006, "bounds": [0.000, 0.100]}, + {"clipped": False, "edge": None, "mode": 0.031, "std": 0.005, "bounds": [0.000, 0.120]}, + ] + + def make_fit(diagnostics): + fit = types.SimpleNamespace( + parameters={ + "rprs": diagnostics["mode"], + "tmid": 0.0, + "inc": 89.0, + "a2": 0.0, + } + ) + + def get_parameter_posterior_recenter_diagnostics(key): + assert key == "rprs" + return dict(diagnostics) + + fit.get_parameter_posterior_recenter_diagnostics = get_parameter_posterior_recenter_diagnostics + return fit + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + call_index = len(captured["calls"]) + captured["calls"].append({ + "prior": dict(call_prior), + "duration_prior": duration_prior, + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + return make_fit(diagnostics_sequence[call_index]) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 7), + np.ones(7, dtype=float), + np.full(7, 0.01, dtype=float), + np.ones(7, dtype=float), + {"tmid": 0.0, "rprs": 0.1, "inc": 89.0, "a2": 0.0}, + {"rprs": [0.025, 0.300], "tmid": [-0.01, 0.01], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]}, + ) + + assert len(captured["calls"]) == 2 + assert captured["calls"][0]["bounds"]["rprs"] == pytest.approx([0.025, 0.300]) + assert captured["calls"][1]["prior"]["rprs"] == pytest.approx(0.030) + assert captured["calls"][1]["bounds"]["rprs"] == pytest.approx([0.000, 0.300]) + assert fit.rprs_posterior_refit_applied is True + assert fit.rprs_posterior_refit_count == 1 + assert fit.rprs_posterior_refit_edge == "lower" + assert fit.rprs_posterior_refit_bounds == pytest.approx([0.000, 0.300]) + + +def test_ars_posterior_retry_expands_bounds_when_upper_edge_is_truncated(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + diagnostics_sequence = [ + { + "rprs": {"clipped": False, "edge": None, "mode": 0.1, "std": 0.01, "bounds": [0.05, 0.15]}, + "ars": {"clipped": True, "edge": "upper", "mode": 14.45, "std": 0.37, "bounds": [12.60, 16.30]}, + }, + { + "rprs": {"clipped": False, "edge": None, "mode": 0.1, "std": 0.01, "bounds": [0.05, 0.15]}, + "ars": {"clipped": False, "edge": None, "mode": 14.50, "std": 0.20, "bounds": [12.60, 16.30]}, + }, + ] + + def make_fit(diagnostics): + fit = types.SimpleNamespace( + parameters={ + "rprs": diagnostics["rprs"]["mode"], + "ars": diagnostics["ars"]["mode"], + "tmid": 0.0, + "inc": 89.0, + "a2": 0.0, + } + ) + + def get_parameter_posterior_recenter_diagnostics(key): + return dict(diagnostics[key]) + + fit.get_parameter_posterior_recenter_diagnostics = get_parameter_posterior_recenter_diagnostics + return fit + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + call_index = len(captured["calls"]) + captured["calls"].append({ + "prior": dict(call_prior), + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + return make_fit(diagnostics_sequence[call_index]) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 7), + np.ones(7, dtype=float), + np.full(7, 0.01, dtype=float), + np.ones(7, dtype=float), + {"tmid": 0.0, "rprs": 0.1, "ars": 14.0, "inc": 89.0, "a2": 0.0}, + { + "rprs": [0.0, 0.25], + "ars": [12.80, 14.80], + "tmid": [-0.01, 0.01], + "inc": [84.0, 90.0], + "a2": [-3.0, 3.0], + }, + ) + + assert len(captured["calls"]) == 2 + assert captured["calls"][0]["bounds"]["ars"] == pytest.approx([12.80, 14.80]) + assert captured["calls"][1]["prior"]["ars"] == pytest.approx(14.45) + assert captured["calls"][1]["bounds"]["ars"] == pytest.approx([12.60, 16.30]) + assert fit.rprs_posterior_refit_applied is False + assert fit.ars_posterior_refit_applied is True + assert fit.ars_posterior_refit_count == 1 + assert fit.ars_posterior_refit_edge == "upper" + assert fit.ars_posterior_refit_bounds == pytest.approx([12.60, 16.30]) + + +def test_toi_tic_ars_posterior_can_retry_beyond_established_target_limit(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_ENABLED", True) + monkeypatch.setattr( + exotic_module, + "ARS_RANGE_RESTRICTION_PERCENTAGE", + ARS_RANGE_RESTRICTION_PERCENTAGE_DEFAULT, + ) + captured_calls = [] + proposed_bounds = [ + [6.8, 13.2], + [6.6, 13.4], + [6.4, 13.6], + [6.2, 13.8], + [6.0, 14.0], + [5.8, 14.2], + ] + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + **kwargs, + ): + call_index = len(captured_calls) + captured_calls.append({ + "prior": dict(call_prior), + "bounds": {key: list(value) for key, value in call_bounds.items()}, + }) + clipped = call_index < len(proposed_bounds) + ars_bounds = proposed_bounds[call_index] if clipped else list(call_bounds["ars"]) + fit = types.SimpleNamespace( + parameters={"rprs": 0.1, "ars": 10.0, "tmid": 0.0, "inc": 89.0, "a2": 0.0}, + ) + + def diagnostics(key): + if key == "rprs": + return {"clipped": False, "edge": None, "mode": 0.1, "std": 0.01, "bounds": [0.05, 0.15]} + return { + "clipped": clipped, + "edge": "upper" if clipped else None, + "mode": 10.0, + "std": 0.2, + "bounds": ars_bounds, + } + + fit.get_parameter_posterior_recenter_diagnostics = diagnostics + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 7), + np.ones(7, dtype=float), + np.full(7, 0.01, dtype=float), + np.ones(7, dtype=float), + {"tmid": 0.0, "rprs": 0.1, "ars": 10.0, "inc": 89.0, "a2": 0.0}, + { + "rprs": [0.05, 0.15], + "ars": [7.0, 13.0], + "tmid": [-0.01, 0.01], + "inc": [84.0, 90.0], + "a2": [-3.0, 3.0], + }, + search_restriction_prior={ + "pName": "TOI-2969 b", + "sName": "TOI-2969", + "ars": 10.0, + "ars_unc": 0.2, + }, + ) + + assert len(captured_calls) == 7 + assert captured_calls[0]["bounds"]["ars"] == pytest.approx([7.0, 13.0]) + assert captured_calls[-1]["bounds"]["ars"] == pytest.approx([5.8, 14.2]) + assert fit.ars_posterior_refit_count == 6 + + +def test_ars_posterior_pinned_at_configured_restriction_falls_back_to_prior(monkeypatch): + import exotic.exotic as exotic_module + + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_ENABLED", True) + monkeypatch.setattr(exotic_module, "ARS_RANGE_RESTRICTION_PERCENTAGE", 10.0) + captured = {"calls": []} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + fixed_parameter_errors=None, + **kwargs, + ): + captured["calls"].append({ + "prior": dict(call_prior), + "bounds": dict(call_bounds), + "fixed_parameter_errors": dict(fixed_parameter_errors or {}), + }) + parameters = dict(call_prior) + if "ars" in call_bounds: + parameters["ars"] = 10.99 + fit = types.SimpleNamespace( + parameters=parameters, + errors=dict(fixed_parameter_errors or {}), + sampled_keys=list(call_bounds), + sample_bounds=dict(call_bounds), + ) + + def diagnostics(key): + if key == "ars": + return { + "clipped": True, + "edge": "upper", + "mode": 10.99, + "std": 0.20, + "bounds": [9.0, 12.0], + } + return { + "clipped": False, + "edge": None, + "mode": parameters.get(key, np.nan), + "std": 0.01, + "bounds": call_bounds.get(key), + "reason": "posterior is not clipped", + } + + fit.get_parameter_posterior_recenter_diagnostics = diagnostics + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 7), + np.ones(7, dtype=float), + np.full(7, 0.01, dtype=float), + np.ones(7, dtype=float), + {"tmid": 0.0, "rprs": 0.1, "ars": 10.0, "inc": 89.0, "a2": 0.0}, + { + "rprs": [0.0, 0.25], + "ars": [9.0, 11.0], + "tmid": [-0.01, 0.01], + "inc": [84.0, 90.0], + "a2": [-3.0, 3.0], + }, + search_restriction_prior={ + "rprs": 0.1, + "ars": 10.0, + "ars_unc": 0.4, + "inc": 89.0, + }, + ) + + assert len(captured["calls"]) == 3 + assert captured["calls"][0]["bounds"]["ars"] == pytest.approx([9.0, 11.0]) + assert captured["calls"][1]["bounds"]["ars"] == pytest.approx([9.0, 12.0]) + assert "ars" not in captured["calls"][2]["bounds"] + assert captured["calls"][2]["prior"]["ars"] == pytest.approx(10.0) + assert captured["calls"][2]["fixed_parameter_errors"]["ars"] == pytest.approx(0.4) + assert fit.parameters["ars"] == pytest.approx(10.0) + assert fit.errors["ars"] == pytest.approx(0.4) + assert fit.ars_prior_fallback_applied is True + assert "could not widen the sampled bounds" in fit.ars_prior_fallback_note + + +def test_partial_coverage_suppresses_open_geometry_posterior_retries(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + diagnostics = { + "rprs": {"clipped": False, "edge": None, "mode": 0.1, "std": 0.01, "bounds": [0.05, 0.15]}, + "ars": {"clipped": True, "edge": "lower", "mode": 5.0, "std": 3.0, "bounds": [0.000001, 20.0]}, + "b": {"clipped": True, "edge": "upper", "mode": 1.6, "std": 0.3, "bounds": [0.5, 2.5]}, + } + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + captured["calls"].append({"prior": dict(call_prior), "bounds": dict(call_bounds)}) + fit = types.SimpleNamespace( + sampled_keys=["rprs", "ars", "b", "tmid"], + sample_bounds={"rprs": [0.0, 0.25], "ars": [0.000001, 20.0], "b": [0.0, 2.5]}, + parameters={"rprs": 0.1, "ars": 5.0, "tmid": 0.0, "inc": 80.0, "a2": 0.0}, + ) + fit.get_parameter_posterior_recenter_diagnostics = lambda key: dict(diagnostics[key]) + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 7), + np.ones(7, dtype=float), + np.full(7, 0.01, dtype=float), + np.ones(7, dtype=float), + {"tmid": 0.0, "rprs": 0.1, "ars": 10.0, "inc": 89.0, "a2": 0.0}, + { + "rprs": [0.0, 0.25], + "ars": [5.0, 15.0], + "tmid": [-0.01, 0.01], + "inc": [70.0, 90.0], + "a2": [-3.0, 3.0], + }, + pre_ultranest_coverage_assessment={ + "valid": True, + "success_label": "low", + "expected_successful": False, + "pre_ingress_points": 0, + "post_egress_points": 8, + }, + ) + + assert len(captured["calls"]) == 1 + assert fit.ars_posterior_refit_applied is False + assert "one-sided/LOW" in fit.ars_posterior_refit_note + assert fit.b_posterior_refit_applied is False + assert "one-sided/LOW" in fit.b_posterior_refit_note + + +def test_one_sided_partial_coverage_fixes_geometry_and_samples_tmid_only(monkeypatch): + import exotic.exotic as exotic_module + + captured = {} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + fixed_parameter_errors=None, + fixed_flux_baseline=False, + **kwargs, + ): + captured["prior"] = dict(call_prior) + captured["bounds"] = dict(call_bounds) + captured["fixed_parameter_errors"] = dict(fixed_parameter_errors or {}) + captured["fixed_flux_baseline"] = bool(fixed_flux_baseline) + fit = types.SimpleNamespace( + time=np.asarray(call_times, dtype=float), + data=np.asarray(call_flux, dtype=float), + dataerr=np.asarray(call_fluxerr, dtype=float), + airmass=np.asarray(call_airmass, dtype=float), + sampled_keys=list(call_bounds.keys()), + sample_bounds=dict(call_bounds), + parameters=dict(call_prior), + errors=dict(fixed_parameter_errors or {}), + transit=np.ones(len(call_times), dtype=float), + residuals=np.zeros(len(call_times), dtype=float), + ) + fit.get_parameter_posterior_recenter_diagnostics = ( + lambda key: {"clipped": False, "reason": "parameter was fixed to the prior"} + ) + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.06, 0.02, 20), + np.ones(20, dtype=float), + np.full(20, 0.01, dtype=float), + np.linspace(1.0, 1.4, 20), + { + "tmid": 0.0, + "rprs": 0.1, + "ars": 12.0, + "inc": 89.0, + "per": 1.0, + "ecc": 0.0, + "omega": 0.0, + "a0": 1.0, + "a2": 0.1, + }, + { + "rprs": [0.0, 0.2], + "tmid": [-0.05, 0.05], + "ars": [10.0, 14.0], + "inc": [84.0, 90.0], + "a0": [0.95, 1.05], + "a2": [-3.0, 3.0], + }, + pre_ultranest_coverage_assessment={ + "valid": True, + "transit_fraction_observed": 0.55, + "in_transit_points": 12, + "pre_ingress_points": 5, + "post_egress_points": 0, + "observed_segment": "pre-ingress baseline plus ingress plus mid-transit", + }, + search_restriction_prior={"rprs_unc": 0.002, "ars_unc": 0.3, "inc_unc": 0.4}, + ) + + assert list(captured["bounds"]) == ["tmid"] + assert captured["fixed_flux_baseline"] is False + assert captured["fixed_parameter_errors"]["rprs"] == pytest.approx(0.002) + assert captured["fixed_parameter_errors"]["ars"] == pytest.approx(0.3) + assert captured["fixed_parameter_errors"]["inc"] == pytest.approx(0.4) + assert captured["fixed_parameter_errors"]["a2"] == pytest.approx(0.025) + assert fit.partial_transit_geometry_prior_assumption_applied is True + assert fit.partial_transit_geometry_prior_assumption_mode == "tmid_only" + assert fit.partial_transit_geometry_prior_assumption_sampled_parameters == ["tmid"] + + +def test_no_oot_partial_coverage_keeps_baseline_airmass_in_ultranest(monkeypatch): + import exotic.exotic as exotic_module + + captured = {} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + fixed_parameter_errors=None, + fixed_flux_baseline=False, + **kwargs, + ): + captured["prior"] = dict(call_prior) + captured["bounds"] = dict(call_bounds) + captured["fixed_parameter_errors"] = dict(fixed_parameter_errors or {}) + captured["fixed_flux_baseline"] = bool(fixed_flux_baseline) + fit = types.SimpleNamespace( + time=np.asarray(call_times, dtype=float), + data=np.asarray(call_flux, dtype=float), + dataerr=np.asarray(call_fluxerr, dtype=float), + airmass=np.asarray(call_airmass, dtype=float), + sampled_keys=list(call_bounds.keys()), + sample_bounds=dict(call_bounds), + parameters=dict(call_prior), + errors=dict(fixed_parameter_errors or {}), + transit=np.ones(len(call_times), dtype=float), + residuals=np.zeros(len(call_times), dtype=float), + ) + fit.get_parameter_posterior_recenter_diagnostics = ( + lambda key: {"clipped": False, "reason": "parameter was fixed to the prior"} + ) + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 20), + np.full(20, 1.02, dtype=float), + np.full(20, 0.01, dtype=float), + np.linspace(1.0, 1.5, 20), + { + "tmid": 0.0, + "rprs": 0.1, + "ars": 12.0, + "inc": 89.0, + "per": 1.0, + "ecc": 0.0, + "omega": 0.0, + "a0": 1.02, + "a2": 0.1, + }, + { + "rprs": [0.0, 0.2], + "tmid": [-0.05, 0.05], + "ars": [10.0, 14.0], + "inc": [84.0, 90.0], + }, + fixed_flux_baseline=True, + pre_ultranest_coverage_assessment={ + "valid": True, + "transit_fraction_observed": 0.90, + "in_transit_points": 18, + "pre_ingress_points": 0, + "post_egress_points": 0, + "observed_segment": "inside the expected transit", + }, + search_restriction_prior={"rprs_unc": 0.002, "ars_unc": 0.3, "inc_unc": 0.4}, + ) + + assert "rprs" not in captured["bounds"] + assert "ars" not in captured["bounds"] + assert "inc" not in captured["bounds"] + assert set(captured["bounds"]) == {"tmid", "a0", "a2"} + assert captured["fixed_flux_baseline"] is False + assert captured["fixed_parameter_errors"]["rprs"] == pytest.approx(0.002) + assert captured["fixed_parameter_errors"]["ars"] == pytest.approx(0.3) + assert captured["fixed_parameter_errors"]["inc"] == pytest.approx(0.4) + assert fit.partial_transit_geometry_prior_assumption_applied is True + assert fit.partial_transit_geometry_prior_assumption_mode == "tmid_baseline_airmass" + assert set(fit.partial_transit_geometry_prior_assumption_sampled_parameters) == {"tmid", "a0", "a2"} + + +def test_impact_parameter_posterior_retry_expands_inclination_bounds(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + diagnostics_sequence = [ + { + "rprs": {"clipped": False, "edge": None, "mode": 0.1, "std": 0.01, "bounds": [0.05, 0.15]}, + "b": {"clipped": True, "edge": "upper", "mode": 1.6, "std": 0.18, "bounds": [0.7, 2.5]}, + }, + { + "rprs": {"clipped": False, "edge": None, "mode": 0.1, "std": 0.01, "bounds": [0.05, 0.15]}, + "b": {"clipped": False, "edge": None, "mode": 1.6, "std": 0.12, "bounds": [0.7, 2.5]}, + }, + ] + + def make_fit(diagnostics): + fit = types.SimpleNamespace( + sampled_keys=["rprs", "b", "tmid"], + sample_bounds={"rprs": [0.0, 0.25], "b": [0.0, 2.5], "tmid": [-0.01, 0.01]}, + parameters={ + "rprs": diagnostics["rprs"]["mode"], + "ars": 10.0, + "tmid": 0.0, + "inc": 80.5, + "a2": 0.0, + }, + ) + + def get_parameter_posterior_recenter_diagnostics(key): + return dict(diagnostics[key]) + + fit.get_parameter_posterior_recenter_diagnostics = get_parameter_posterior_recenter_diagnostics + return fit + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + call_index = len(captured["calls"]) + captured["calls"].append({ + "prior": dict(call_prior), + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + "use_impactparameter": use_impactparameter_rather_than_inclination_to_fit, + }) + return make_fit(diagnostics_sequence[call_index]) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 7), + np.ones(7, dtype=float), + np.full(7, 0.01, dtype=float), + np.ones(7, dtype=float), + {"tmid": 0.0, "rprs": 0.1, "ars": 10.0, "inc": 85.0, "a2": 0.0}, + {"rprs": [0.0, 0.25], "tmid": [-0.01, 0.01], "inc": [80.0, 90.0], "a2": [-3.0, 3.0]}, + ) + + assert len(captured["calls"]) == 2 + assert captured["calls"][0]["bounds"]["inc"] == pytest.approx([80.0, 90.0]) + assert captured["calls"][1]["bounds"]["inc"][0] == pytest.approx(np.degrees(np.arccos(0.25))) + assert captured["calls"][1]["bounds"]["inc"][1] == pytest.approx(90.0) + assert captured["calls"][1]["prior"]["inc"] == pytest.approx(80.5) + assert fit.b_posterior_refit_applied is True + assert fit.b_posterior_refit_count == 1 + assert fit.b_posterior_refit_edge == "upper" + assert fit.b_posterior_refit_bounds == pytest.approx([np.degrees(np.arccos(0.25)), 90.0]) + + +def test_impact_parameter_retry_is_skipped_when_b_is_sampled_directly(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + diagnostics = { + "rprs": {"clipped": False, "edge": None, "mode": 0.1, "std": 0.01, "bounds": [0.05, 0.15]}, + "b": {"clipped": True, "edge": "upper", "mode": 1.1, "std": 0.04, "bounds": [0.0, 1.12]}, + } + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + captured["calls"].append({"prior": dict(call_prior), "bounds": dict(call_bounds)}) + fit = types.SimpleNamespace( + sampled_keys=["rprs", "b", "tmid"], + sample_bounds={"rprs": [0.0, 0.25], "b": [0.0, 1.12], "tmid": [-0.01, 0.01]}, + impact_parameter_sampled_directly=True, + parameters={"rprs": 0.1, "ars": 10.0, "tmid": 0.0, "inc": 83.5, "a2": 0.0}, + ) + fit.get_parameter_posterior_recenter_diagnostics = lambda key: dict(diagnostics[key]) + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 7), + np.ones(7, dtype=float), + np.full(7, 0.01, dtype=float), + np.ones(7, dtype=float), + {"tmid": 0.0, "rprs": 0.1, "ars": 10.0, "inc": 85.0, "a2": 0.0}, + {"rprs": [0.0, 0.25], "tmid": [-0.01, 0.01], "inc": [80.0, 90.0], "a2": [-3.0, 3.0]}, + ) + + assert len(captured["calls"]) == 1 + assert fit.b_posterior_refit_applied is False + assert fit.b_posterior_refit_count == 0 + + +def test_impact_parameter_posterior_retry_expands_toward_face_on_boundary(monkeypatch): + import exotic.exotic as exotic_module + + captured = {"calls": []} + diagnostics_sequence = [ + { + "rprs": {"clipped": False, "edge": None, "mode": 0.1, "std": 0.01, "bounds": [0.05, 0.15]}, + "b": {"clipped": True, "edge": "lower", "mode": 0.55, "std": 0.10, "bounds": [0.0, 1.0]}, + }, + { + "rprs": {"clipped": False, "edge": None, "mode": 0.1, "std": 0.01, "bounds": [0.05, 0.15]}, + "b": {"clipped": False, "edge": None, "mode": 0.55, "std": 0.08, "bounds": [0.0, 1.0]}, + }, + ] + + def make_fit(diagnostics): + fit = types.SimpleNamespace( + sampled_keys=["rprs", "b", "tmid"], + sample_bounds={"rprs": [0.0, 0.25], "b": [0.0, 1.0], "tmid": [-0.01, 0.01]}, + parameters={"rprs": 0.1, "ars": 10.0, "tmid": 0.0, "inc": 86.0, "a2": 0.0}, + ) + fit.get_parameter_posterior_recenter_diagnostics = lambda key: dict(diagnostics[key]) + return fit + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + call_index = len(captured["calls"]) + captured["calls"].append({ + "prior": dict(call_prior), + "bounds": { + key: list(value) if isinstance(value, (list, tuple, np.ndarray)) else value + for key, value in call_bounds.items() + }, + }) + return make_fit(diagnostics_sequence[call_index]) + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 7), + np.ones(7, dtype=float), + np.full(7, 0.01, dtype=float), + np.ones(7, dtype=float), + {"tmid": 0.0, "rprs": 0.1, "ars": 10.0, "inc": 84.0, "a2": 0.0}, + {"rprs": [0.0, 0.25], "tmid": [-0.01, 0.01], "inc": [80.0, 87.0], "a2": [-3.0, 3.0]}, + ) + + assert len(captured["calls"]) == 2 + assert captured["calls"][1]["bounds"]["inc"] == pytest.approx([80.0, 90.0]) + assert fit.b_posterior_refit_applied is True + assert fit.b_posterior_refit_edge == "lower" + + +def test_run_nested_lightcurve_fit_passes_duration_prior_when_available(monkeypatch): + import exotic.exotic as exotic_module + + captured = {} + diagnostics = {"clipped": False, "edge": None, "mode": 0.1, "std": 0.01, "bounds": [0.05, 0.15]} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + ): + captured["duration_prior"] = duration_prior + fit = types.SimpleNamespace(parameters={"rprs": 0.1, "ars": 15.0, "tmid": 0.0, "inc": 89.0, "a2": 0.0}) + + def get_parameter_posterior_recenter_diagnostics(key): + assert key in ("rprs", "ars") + return dict(diagnostics) + + fit.get_parameter_posterior_recenter_diagnostics = get_parameter_posterior_recenter_diagnostics + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + duration_prior = build_single_transit_duration_prior({ + "pPer": 1.0, + "pPerUnc": 0.001, + "rprs": 0.1, + "rprsUnc": 0.01, + "aRs": 15.0, + "aRsUnc": 0.1, + "inc": 89.0, + "incUnc": 0.1, + "ecc": 0.0, + "omega": 0.0, + }) + + fit = run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 7), + np.ones(7, dtype=float), + np.full(7, 0.01, dtype=float), + np.ones(7, dtype=float), + {"tmid": 0.0, "rprs": 0.1, "ars": 15.0, "inc": 89.0, "a2": 0.0}, + {"rprs": [0.0, 0.25], "ars": [14.5, 15.5], "tmid": [-0.01, 0.01], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]}, + duration_prior=duration_prior, + ) + + assert captured["duration_prior"] == duration_prior + assert fit.duration_prior_applied is True + assert "expected duration=" in fit.duration_prior_note + + +def test_sparse_posterior_metric_flags_under_sampled_key_parameters(): + fit = types.SimpleNamespace() + fit.get_parameter_posterior_samples = lambda key: np.linspace(0.0, 1.0, 100) + + diagnostics = evaluate_sparse_posterior_sample_support( + fit, + base_live_points=200, + ) + + assert diagnostics["sparse"] is True + assert diagnostics["minimum_effective_samples"] == 1000 + assert diagnostics["parameters"]["rprs"]["effective_sample_count"] == pytest.approx(100) + assert "rprs" in diagnostics["reason"] + + +def test_sparse_posterior_extension_continues_existing_ultranest_sampler(monkeypatch): + monkeypatch.setenv("EXOTIC_ULTRANEST_MIN_NUM_LIVE_POINTS", "200") + + class SparseFit: + def __init__(self): + self.samples = { + "rprs": np.linspace(0.09, 0.11, 100), + "tmid": np.linspace(-0.001, 0.001, 100), + "ars": np.linspace(9.5, 10.5, 100), + } + self.max_ncalls = 1000 + self.extension_calls = [] + self.cleared = False + + def get_parameter_posterior_samples(self, key): + return self.samples[key] + + def extend_ultranest_fit(self, min_num_live_points=None, max_ncalls=None): + self.extension_calls.append({ + "min_num_live_points": min_num_live_points, + "max_ncalls": max_ncalls, + }) + self.samples = { + "rprs": np.linspace(0.09, 0.11, 1000), + "tmid": np.linspace(-0.001, 0.001, 1000), + "ars": np.linspace(9.5, 10.5, 1000), + } + return True + + def clear_ultranest_resume_state(self): + self.cleared = True + + fit = SparseFit() + returned = extend_sparse_posterior_live_points_if_needed( + fit, + enabled=True, + extension_factor=SPARSE_POSTERIOR_LIVE_POINT_RETRY_FACTOR_DEFAULT, + ) + + assert returned is fit + assert fit.extension_calls == [{ + "min_num_live_points": 1200, + "max_ncalls": 6000, + }] + assert fit.sparse_posterior_live_point_extension_applied is True + assert "200->1200" in fit.sparse_posterior_live_point_extension_note + assert fit.cleared is True + + +def test_run_nested_lightcurve_fit_can_retain_sampler_for_final_extension(monkeypatch): + import exotic.exotic as exotic_module + + captured = {} + diagnostics = {"clipped": False, "edge": None, "mode": 0.1, "std": 0.01, "bounds": [0.05, 0.15]} + + def fake_lc_fitter( + call_times, + call_flux, + call_fluxerr, + call_airmass, + call_prior, + call_bounds, + jd_times=None, + mode=None, + use_impactparameter_rather_than_inclination_to_fit=True, + duration_prior=None, + keep_ultranest_sampler=False, + ): + captured["keep_ultranest_sampler"] = keep_ultranest_sampler + fit = types.SimpleNamespace(parameters={"rprs": 0.1, "ars": 15.0, "tmid": 0.0, "inc": 89.0, "a2": 0.0}) + fit.get_parameter_posterior_recenter_diagnostics = lambda key: dict(diagnostics) + return fit + + monkeypatch.setattr(exotic_module, "lc_fitter", fake_lc_fitter) + + run_nested_lightcurve_fit_with_rprs_posterior_retry( + np.linspace(-0.03, 0.03, 7), + np.ones(7, dtype=float), + np.full(7, 0.01, dtype=float), + np.ones(7, dtype=float), + {"tmid": 0.0, "rprs": 0.1, "ars": 15.0, "inc": 89.0, "a2": 0.0}, + {"rprs": [0.0, 0.25], "ars": [14.5, 15.5], "tmid": [-0.01, 0.01], "inc": [84.0, 90.0], "a2": [-3.0, 3.0]}, + keep_ultranest_sampler=True, + ) + + assert captured["keep_ultranest_sampler"] is True diff --git a/tests/test_inputs.py b/tests/test_inputs.py new file mode 100644 index 00000000..41bcb21e --- /dev/null +++ b/tests/test_inputs.py @@ -0,0 +1,2046 @@ +import json +import requests +import pytest +import numpy as np +from astropy.io import fits + +import exotic.inputs as inputs_module +from exotic.inputs import Inputs, camera, parse_aavso_prereduced_overrides + + +def test_camera_accepts_cmos_as_ccd_without_prompt(): + assert camera("CMOS") == "CCD" + + +def test_camera_defaults_to_ccd_when_missing_or_unrecognized(): + assert camera(None) == "CCD" + assert camera("") == "CCD" + assert camera("mirrorless") == "CCD" + + +def test_camera_keeps_dslr_as_dslr(): + assert camera("DSLR") == "DSLR" + assert camera("canon dslr") == "DSLR" + + +@pytest.mark.parametrize("parser", [inputs_module.plate_solution_opt, inputs_module.aavso_comp]) +def test_user_info_boolean_options_accept_supported_forms_without_prompt(monkeypatch, parser): + monkeypatch.setattr( + inputs_module, + "user_input", + lambda *_args, **_kwargs: pytest.fail("valid boolean initialization value must not prompt"), + ) + + for value in (True, 1, "1", "y", "Y", "yes", "TRUE", "on"): + assert parser(value) == "y" + + for value in (False, 0, "0", "n", "N", "no", "FALSE", "off"): + assert parser(value) == "n" + + +def test_comparison_star_coords_accepts_more_than_ten_manual_comps(): + comp_stars = [[float(index), float(index + 1)] for index in range(12)] + + assert inputs_module.comparison_star_coords(comp_stars, rt_bool=False) == comp_stars + + +def test_comparison_star_radec_coords_accepts_decimal_and_sexagesimal_pairs(): + coords = inputs_module.comparison_star_radec_coords([ + [31.04125, 46.68972], + ["02:04:09.90", "+46:41:23.0"], + [], + ]) + + assert coords[0] == pytest.approx([31.04125, 46.68972]) + assert coords[1][0] == pytest.approx(31.04125) + assert coords[1][1] == pytest.approx(46.6897222222) + + +def test_comp_params_accepts_comparison_radec_without_pixel_coordinates(tmp_path): + init_data = { + "user_info": { + "Comparison Star(s) X & Y Pixel": [], + "Comparison Star(s) RA & Dec": [[31.04125, 46.68972]], + }, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data), encoding="utf-8") + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert np.allclose(inputs.info_dict["comp_stars_radec"], [[31.04125, 46.68972]]) + + +def test_comp_params_accepts_sexagesimal_comparison_radec_in_exact_mode(tmp_path): + init_data = { + "user_info": { + "Comparison Star(s) X & Y Pixel": [], + "Comparison Star(s) RA & Dec": [["18:37:32.87", "+18:45:39.4"]], + }, + "optional_info": { + "use_exactly_the_comps_provided": True, + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data), encoding="utf-8") + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert np.allclose( + inputs.info_dict["comp_stars_radec"], + [[279.3869583333, 18.7609444444]], + ) + assert inputs.info_dict["use_exactly_the_comps_provided"] is True + + +def test_comp_params_rejects_simultaneous_pixel_and_radec_comparisons(tmp_path): + init_data = { + "user_info": { + "Comparison Star(s) X & Y Pixel": [[465, 183]], + "Comparison Star(s) RA & Dec": [[31.04125, 46.68972]], + }, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data), encoding="utf-8") + + with pytest.raises(ValueError, match="either .*X & Y Pixel.*or .*RA & Dec.*not both"): + Inputs(init_opt="y").comp_params(init_file, {}) + + +def test_comp_params_accepts_verbose_camera_key(tmp_path): + init_data = { + "user_info": { + "Camera Type (e.g., CCD or DSLR; Note: if you are using a CMOS, please enter CCD here and then note your actual camera type in \"Observing Notes\")": "CCD" + }, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["camera"] == "CCD" + + +def test_comp_params_defaults_require_comp_star_to_yes(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["require_comp_star"] == "y" + + +def test_comp_params_defaults_aavso_comp_to_no(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["aavso_comp"] == "n" + + +def test_comp_params_defaults_stellar_variability_only_to_false(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["stellar_variability_only"] is False + + +def test_comp_params_reads_stellar_variability_only_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": { + "stellar_variability_only": True, + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["stellar_variability_only"] is True + + +def test_comp_params_defaults_apparent_and_exact_comparison_options(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["require_apparent_magnitudes"] is True + assert inputs.info_dict["use_exactly_the_comps_provided"] is False + + +def test_comp_params_reads_apparent_and_exact_comparison_options(tmp_path): + init_data = { + "user_info": {}, + "optional_info": { + "require_apparent_magnitudes": False, + "use_exactly_the_comps_provided": True, + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["require_apparent_magnitudes"] is False + assert inputs.info_dict["use_exactly_the_comps_provided"] is True + + +def test_comp_params_accepts_exact_comparison_option_beside_user_coordinates(tmp_path): + init_data = { + "user_info": { + "Comparison Star(s) RA & Dec": [[279.3869583, 18.7609444]], + "use_exactly_the_comps_provided": True, + }, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data), encoding="utf-8") + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_exactly_the_comps_provided"] is True + + +def test_optional_info_exact_comparison_option_overrides_user_info_alias(tmp_path): + init_data = { + "user_info": { + "Comparison Star(s) RA & Dec": [[279.3869583, 18.7609444]], + "use_exactly_the_comps_provided": True, + }, + "optional_info": { + "use_exactly_the_comps_provided": False, + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data), encoding="utf-8") + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_exactly_the_comps_provided"] is False + + +def test_comp_params_defaults_stellar_variability_ensemble_to_true(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_ensemble_photometry_for_stellar_variability"] is True + + +def test_comp_params_reads_stellar_variability_ensemble_opt_out(tmp_path): + init_data = { + "user_info": {}, + "optional_info": { + "use_ensemble_photometry_for_stellar_variability": False, + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_ensemble_photometry_for_stellar_variability"] is False + + +def test_comp_params_defaults_independent_ensemble_comparison_limits_to_five(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["maximum_number_of_ensemble_comparisons_for_transit"] == 5 + assert ( + inputs.info_dict["maximum_number_of_ensemble_comparisons_for_stellar_variability"] + == 5 + ) + + +def test_comp_params_reads_different_transit_and_variability_ensemble_limits(tmp_path): + init_data = { + "user_info": {}, + "optional_info": { + "maximum_number_of_ensemble_comparisons_for_transit": 250, + "maximum_number_of_ensemble_comparisons_for_stellar_variability": 125, + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["maximum_number_of_ensemble_comparisons_for_transit"] == 250 + assert ( + inputs.info_dict["maximum_number_of_ensemble_comparisons_for_stellar_variability"] + == 125 + ) + + +def test_comp_params_defaults_fortuitous_variable_photometry_to_true(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["photometer_fortuitous_variables"] is True + + +def test_comp_params_reads_fortuitous_variable_photometry_opt_out(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"photometer_fortuitous_variables": False}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["photometer_fortuitous_variables"] is False + + +def test_comp_params_defaults_fortuitous_variables_to_single_comparison(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_single_comparison_for_fortuitous_variables"] is True + + +def test_comp_params_reads_fortuitous_single_comparison_opt_out(tmp_path): + init_data = { + "user_info": {}, + "optional_info": { + "use_single_comparison_for_fortuitous_variables": False, + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_single_comparison_for_fortuitous_variables"] is False + + +def test_comp_params_defaults_nextastro_vsx_cache_first_to_false(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_nextastro_vsx_cache_first"] is False + + +def test_comp_params_reads_nextastro_vsx_cache_first_opt_in(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"use_nextastro_vsx_cache_first": True}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_nextastro_vsx_cache_first"] is True + + +def test_comp_params_defaults_overexposure_rejection_options(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["reject_overexposed_stars"] is True + assert inputs.info_dict["saturation_value"] == pytest.approx(65535.0) + assert inputs.info_dict["overexposure_threshold_fraction"] == pytest.approx(0.9) + + +def test_comp_params_reads_overexposure_rejection_options_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": { + "Reject Overexposed Stars? (y/n)": False, + "Saturation Value": 42000, + "Overexposure Threshold Fraction": 0.75, + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["reject_overexposed_stars"] is False + assert inputs.info_dict["saturation_value"] == pytest.approx(42000.0) + assert inputs.info_dict["overexposure_threshold_fraction"] == pytest.approx(0.75) + + +def test_complete_red_uses_fits_x_y_binning_when_pixel_bin_missing(tmp_path, monkeypatch): + image_dir = tmp_path / "images" + image_dir.mkdir() + save_dir = tmp_path / "results" + save_dir.mkdir() + + header = fits.Header() + header["XBINNING"] = 2 + header["YBINNING"] = 3 + fits.PrimaryHDU(data=np.zeros((2, 2)), header=header).writeto(image_dir / "frame.fits") + + init_data = { + "user_info": { + "Directory with FITS files": str(image_dir), + "Directory to Save Plots": str(save_dir), + "AAVSO Observer Code (blank if none)": "", + "Secondary Observer Codes (blank if none)": "", + "Observation date": "2020-01-01", + "Obs. Latitude": "+32.0", + "Obs. Longitude": "-110.0", + "Obs. Elevation (meters)": 1000, + "Camera Type (CCD or DSLR)": "CCD", + "Observing Notes": "na", + "Plate Solution? (y/n)": "n", + "Add Comparison Stars from AAVSO? (y/n)": "n", + "Target Star X & Y Pixel": [1, 1], + "Comparison Star(s) X & Y Pixel": [[2, 2]], + }, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + def fail_on_prompt(prompt, type_, values=None, max_tries=1000): + raise AssertionError(f"Unexpected prompt: {prompt}") + + monkeypatch.setattr(inputs_module, "user_input", fail_on_prompt) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + info_dict, _ = inputs.complete_red("HAT-P-32 b") + + assert info_dict["pixel_bin"] == "2x3" + + +def test_comp_params_defaults_ignore_header_wcs_to_no(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["ignore_header_wcs"] == "n" + + +def test_comp_params_defaults_allow_pixel_alignment_fallback_to_true(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["allow_pixel_alignment_fallback"] is True + + +def test_comp_params_defaults_prefer_pixel_values_over_wcs_for_target_to_no(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["prefer_pixel_values_over_wcs_for_target"] == "n" + + +def test_comp_params_defaults_bad_wcs_threshold_percent_to_three(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["bad_wcs_threshold_percent"] == 3.0 + + +def test_comp_params_defaults_pointing_rejection_sigma_to_none(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["pointing_rejection_sigma"] is None + + +def test_comp_params_defaults_skip_low_comparison_coverage_rejection_to_no(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["skip_low_comparison_coverage_rejection"] == "n" + + +def test_comp_params_defaults_fit_lightcurve_to_every_comparison_candidate_to_no(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["fit_lightcurve_to_every_comparison_candidate"] == "n" + + +def test_comp_params_defaults_ultranest_live_points_to_200(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["ultranest_min_num_live_points"] == 200 + + +def test_comp_params_defaults_rprs_search_bound_max_to_half(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["rprs_search_bound_max"] == 0.5 + + +def test_comp_params_defaults_prior_centered_search_restrictions_to_on(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["restrict_rprs_range"] == "y" + assert inputs.info_dict["restrict_rprs_range_percentage"] == 10.0 + assert inputs.info_dict["restrict_ars_range"] == "y" + assert inputs.info_dict["restrict_ars_range_percentage"] == 10.0 + + +def test_comp_params_defaults_sparse_posterior_live_point_retry_to_yes(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_sparse_posterior_live_point_retry"] == "y" + + +def test_comp_params_defaults_exit_at_first_qc_pass_solution_to_yes(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["exit_at_first_qc_pass_solution"] == "y" + + +def test_comp_params_defaults_disable_vertical_flux_normalization_to_false(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["disable_vertical_flux_normalization"] is False + + +def test_comp_params_defaults_detect_bad_pixels_before_photometry_to_no(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["detect_bad_pixels_before_photometry"] == "n" + + +def test_comp_params_defaults_multiprocess_bad_pixel_precheck_to_no(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["multiprocess_bad_pixel_precheck"] == "n" + + +def test_comp_params_defaults_detrend_on_outoftransit_baseline_to_true(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["detrend_on_outoftransit_baseline"] is True + + +def test_comp_params_defaults_final_fit_baseline_duration_multiplier_to_one(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["final_fit_baseline_duration_multiplier"] == pytest.approx(1.0) + + +def test_comp_params_defaults_use_eebls_tmid_initializer_to_yes(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_eebls_to_initialize_tmid_and_bounds"] == "y" + + +def test_comp_params_defaults_use_impactparameter_fit_to_yes(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_impactparameter_rather_than_inclination_to_fit"] == "y" + + +def test_comp_params_defaults_use_psf_photometry_to_yes(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_psf_photometry"] == "y" + + +def test_comp_params_defaults_use_aperture_photometry_to_yes(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_aperture_photometry"] == "y" + + +def test_comp_params_defaults_fast_aperture_mask_to_false(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["fast_aperture_mask"] is False + + +def test_comp_params_defaults_use_adaptive_apertures_to_false(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_adaptive_apertures"] is False + + +def test_comp_params_defaults_aperture_corrections_and_full_image_fwhm_to_false(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_aperture_corrections_and_full_image_fwhm"] is False + + +def test_comp_params_reads_observatory_full_title_from_user_info(tmp_path): + init_data = { + "user_info": {"Observatory Full Title": "Whipple Observatory"}, + "optional_info": {}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["obs_name"] == "Whipple Observatory" + + +def test_comp_params_reads_require_comp_star_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"require_comp_star": "n"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["require_comp_star"] == "n" + + +def test_comp_params_reads_ignore_header_wcs_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"Ignore WCS in Header and Do Manual Alignment? (y/n)": "y"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["ignore_header_wcs"] == "y" + + +def test_comp_params_reads_allow_pixel_alignment_fallback_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"allow_pixel_alignment_fallback": True}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["allow_pixel_alignment_fallback"] is True + + +def test_comp_params_reads_prefer_pixel_values_over_wcs_for_target_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"prefer_pixel_values_over_wcs_for_target": "y"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["prefer_pixel_values_over_wcs_for_target"] == "y" + + +def test_comp_params_reads_bad_wcs_threshold_percent_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"bad_wcs_threshold_percent": 5.5}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["bad_wcs_threshold_percent"] == 5.5 + + +def test_comp_params_reads_pointing_rejection_sigma_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"pointing_rejection_sigma": 3.5}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["pointing_rejection_sigma"] == 3.5 + + +def test_comp_params_reads_skip_low_comparison_coverage_rejection_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"skip_low_comparison_coverage_rejection": "y"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["skip_low_comparison_coverage_rejection"] == "y" + + +def test_comp_params_reads_fit_lightcurve_to_every_comparison_candidate_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"fit_lightcurve_to_every_comparison_candidate": "y"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["fit_lightcurve_to_every_comparison_candidate"] == "y" + + +def test_comp_params_reads_ultranest_live_points_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"minimum number of live points for ultranest": 275}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["ultranest_min_num_live_points"] == 275 + + +def test_comp_params_reads_rprs_search_bound_max_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"maximum Rp/Rs search bound": 0.35}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["rprs_search_bound_max"] == 0.35 + + +def test_comp_params_reads_prior_centered_search_restrictions_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": { + "restrict_Rp/Rs_range": "n", + "restrict_Rp/Rs_range_percentage": 15, + "restrict_a/Rs_range": "y", + "restrict_a/Rs_range_percentage": "12.5%", + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["restrict_rprs_range"] == "n" + assert inputs.info_dict["restrict_rprs_range_percentage"] == 15 + assert inputs.info_dict["restrict_ars_range"] == "y" + assert inputs.info_dict["restrict_ars_range_percentage"] == "12.5%" + + +def test_comp_params_reads_prior_rprs_fallback_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"Use Prior Rp/Rs When Posterior Pinned? (y/n)": "n"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_prior_rprs_when_posterior_pinned"] == "n" + + +def test_comp_params_reads_fast_ultranest_before_final_run_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"run fast ultranest before final run": "n"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["run_fast_ultranest_before_final_run"] == "n" + + +def test_comp_params_reads_sparse_posterior_live_point_retry_off_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"use_sparse_posterior_live_point_retry": "n"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_sparse_posterior_live_point_retry"] == "n" + + +def test_comp_params_reads_exit_at_first_qc_pass_solution_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"exit at first QC PASS solution": "n"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["exit_at_first_qc_pass_solution"] == "n" + + +def test_comp_params_reads_disable_vertical_flux_normalization_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"disable vertical flux normalization": True}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["disable_vertical_flux_normalization"] is True + + +def test_comp_params_reads_detect_bad_pixels_before_photometry_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"detect_bad_pixels_before_photometry": "n"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["detect_bad_pixels_before_photometry"] == "n" + + +def test_comp_params_reads_multiprocess_bad_pixel_precheck_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"multiprocess_bad_pixel_precheck": "y"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["multiprocess_bad_pixel_precheck"] == "y" + + +def test_comp_params_reads_detrend_on_outoftransit_baseline_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"detrend_on_outoftransit_baseline": True}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["detrend_on_outoftransit_baseline"] is True + + +def test_comp_params_reads_detrend_on_outoftransit_baseline_false_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"detrend_on_outoftransit_baseline": False}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["detrend_on_outoftransit_baseline"] is False + + +def test_comp_params_reads_final_fit_baseline_duration_multiplier_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"final_fit_baseline_duration_multiplier": 1.75}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["final_fit_baseline_duration_multiplier"] == pytest.approx(1.75) + + +def test_comp_params_reads_use_eebls_tmid_initializer_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"Use EEBLS to Initialize Tmid and Bounds? (y/n)": "n"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_eebls_to_initialize_tmid_and_bounds"] == "n" + + +def test_comp_params_reads_use_impactparameter_fit_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"use_impactparameter_rather_than_inclination_to_fit": "n"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_impactparameter_rather_than_inclination_to_fit"] == "n" + + +def test_comp_params_reads_use_psf_photometry_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"use_psf_photometry": "n"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_psf_photometry"] == "n" + + +def test_comp_params_reads_use_legacy_psf_flux_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"use_legacy_psf_flux": "y"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_legacy_psf_flux"] == "y" + + +def test_comp_params_reads_psf_seed_track_directory_from_optional_info(tmp_path): + seed_dir = tmp_path / "old_run" + init_data = { + "user_info": {}, + "optional_info": {"psf_seed_track_directory": str(seed_dir)}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["psf_seed_track_directory"] == str(seed_dir) + + +def test_comp_params_reads_final_fit_phase_residual_clip_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"run_final_fit_phase_residual_clip": "n"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["run_final_fit_phase_residual_clip"] == "n" + + +def test_comp_params_reads_use_aperture_photometry_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"use_aperture_photometry": "n"}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_aperture_photometry"] == "n" + + +def test_comp_params_reads_use_adaptive_apertures_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"use_adaptive_apertures": True}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_adaptive_apertures"] is True + + +def test_comp_params_reads_aperture_corrections_and_full_image_fwhm_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": {"use_aperture_corrections_and_full_image_fwhm": True}, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["use_aperture_corrections_and_full_image_fwhm"] is True + + +def test_comp_params_reads_noise_budget_terms_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": { + "gain_electrons_per_adu": 1.7, + "read_noise_electrons": 5.2, + "dark_current_electrons_per_second_per_pixel": 0.03, + "flat_field_fractional_error": 0.004, + "telescope_aperture_m": 0.28, + "scintillation_coefficient": 0.09, + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["gain_electrons_per_adu"] == pytest.approx(1.7) + assert inputs.info_dict["read_noise_electrons"] == pytest.approx(5.2) + assert inputs.info_dict["dark_current_electrons_per_second_per_pixel"] == pytest.approx(0.03) + assert inputs.info_dict["flat_field_fractional_error"] == pytest.approx(0.004) + assert inputs.info_dict["telescope_aperture_m"] == pytest.approx(0.28) + assert inputs.info_dict["scintillation_coefficient"] == pytest.approx(0.09) + + +def test_comp_params_reads_colour_term_metadata_from_optional_info(tmp_path): + init_data = { + "user_info": {}, + "optional_info": { + "colour_term": -0.045, + "colour_term_error": 0.004, + "colour_term_index": "r-i", + "colour_term_bv": -0.031, + "colour_term_bv_error": 0.003, + "colour_term_bprp": -0.028, + "colour_term_bprp_error": 0.002, + "colour_equation_filter": "rp", + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["colour_term"] == pytest.approx(-0.045) + assert inputs.info_dict["colour_term_error"] == pytest.approx(0.004) + assert inputs.info_dict["colour_term_index"] == "r-i" + assert inputs.info_dict["colour_term_bv"] == pytest.approx(-0.031) + assert inputs.info_dict["colour_term_bv_error"] == pytest.approx(0.003) + assert inputs.info_dict["colour_term_bprp"] == pytest.approx(-0.028) + assert inputs.info_dict["colour_term_bprp_error"] == pytest.approx(0.002) + assert inputs.info_dict["colour_equation_filter"] == "rp" + + +class DummyResponse: + def __init__(self, payload): + self._payload = payload + + def raise_for_status(self): + return None + + def json(self): + return self._payload + + +def test_comp_params_fetches_missing_gaia_astrometry_from_nextastro(tmp_path, monkeypatch): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": { + "Target Star RA": "01:02:03", + "Target Star Dec": "+04:05:06", + "Star Distance (pc)": None, + "Star Proper Motion RA (mas/yr)": "", + "Star Proper Motion DEC (mas/yr)": "null", + }, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + called = {} + + def fake_get(url, params, timeout): + called["url"] = url + called["params"] = params + called["timeout"] = timeout + return DummyResponse({ + "gaia": { + "distance_pc": 200.0, + "pmra_mas_per_year": 10.0, + "pmdec_mas_per_year": -20.0, + } + }) + + monkeypatch.setattr(inputs_module.requests, "get", fake_get) + + inputs = Inputs(init_opt="y") + planet_dict = inputs.comp_params(init_file, {}) + + assert called["url"] == "https://archive.nextastro.org/single_star_gaia_distpm" + assert called["timeout"] == 30 + assert called["params"]["ra"] == pytest.approx(15.5125) + assert called["params"]["dec"] == pytest.approx(4.085) + assert planet_dict["dist"] == 200.0 + assert planet_dict["pm_ra"] == 10.0 + assert planet_dict["pm_dec"] == -20.0 + + +def test_comp_params_only_backfills_missing_gaia_fields(tmp_path, monkeypatch): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": { + "Target Star RA": 123.4501, + "Target Star Dec": -12.3402, + "Star Distance (pc)": 111.0, + "Star Proper Motion RA (mas/yr)": None, + "Star Proper Motion DEC (mas/yr)": "", + }, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + monkeypatch.setattr( + inputs_module.requests, + "get", + lambda url, params, timeout: DummyResponse({ + "gaia": { + "distance_pc": 222.0, + "pmra_mas_per_year": 8.5, + "pmdec_mas_per_year": -4.25, + } + }), + ) + + inputs = Inputs(init_opt="y") + planet_dict = inputs.comp_params(init_file, {}) + + assert planet_dict["dist"] == 111.0 + assert planet_dict["pm_ra"] == 8.5 + assert planet_dict["pm_dec"] == -4.25 + + +def test_comp_params_continues_when_nextastro_gaia_lookup_fails(tmp_path, monkeypatch): + init_data = { + "user_info": {}, + "optional_info": {}, + "planetary_parameters": { + "Target Star RA": 123.4501, + "Target Star Dec": -12.3402, + "Star Distance (pc)": None, + "Star Proper Motion RA (mas/yr)": "", + "Star Proper Motion DEC (mas/yr)": None, + }, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + def raise_request_exception(*args, **kwargs): + raise requests.exceptions.RequestException("service unavailable") + + monkeypatch.setattr(inputs_module.requests, "get", raise_request_exception) + + inputs = Inputs(init_opt="y") + planet_dict = inputs.comp_params(init_file, {}) + + assert planet_dict["dist"] is None + assert planet_dict["pm_ra"] == "" + assert planet_dict["pm_dec"] is None + + +def test_prereduced_mode_forces_aavso_comp_to_no(tmp_path): + pre_reduced_file = tmp_path / "prereduced.txt" + pre_reduced_file.write_text("time flux uncertainty\n") + + inputs = Inputs(init_opt="y") + inputs.info_dict.update({ + "save": str(tmp_path), + "aavso_num": "RTZ", + "second_obs": "", + "date": "2020-01-01", + "lat": "+0.0", + "long": "+0.0", + "elev": 1.0, + "camera": "CCD", + "pixel_bin": "1x1", + "notes": "na", + "aavso_comp": "y", + "prered_file": str(pre_reduced_file), + "exposure": 60.0, + "file_units": "flux", + "file_time": "BJD_TDB", + "phot_comp_star": {"ra": "", "dec": "", "x": "", "y": ""}, + }) + + info_dict, _ = inputs.prereduced("HAT-P-32 b") + + assert info_dict["aavso_comp"] == "n" + + +def test_prereduced_allows_blank_observatory_location_for_bjd_tdb(tmp_path): + pre_reduced_file = tmp_path / "prereduced.txt" + pre_reduced_file.write_text("time flux uncertainty\n") + + inputs = Inputs(init_opt="y") + inputs.info_dict.update({ + "save": str(tmp_path), + "aavso_num": "RTZ", + "second_obs": "", + "date": "2020-01-01", + "lat": "", + "long": "", + "elev": "", + "camera": "CCD", + "pixel_bin": "1x1", + "notes": "na", + "aavso_comp": "y", + "prered_file": str(pre_reduced_file), + "exposure": 60.0, + "file_units": "flux", + "file_time": "BJD_TDB", + "phot_comp_star": None, + }) + + info_dict, _ = inputs.prereduced("HAT-P-32 b") + + assert info_dict["lat"] is None + assert info_dict["long"] is None + assert info_dict["elev"] is None + + +def test_comp_params_accepts_blank_if_none_phot_comp_star_key(tmp_path): + init_data = { + "user_info": {}, + "optional_info": { + "Comparison Star used in Photometry (blank if none)": { + "ra": "", + "dec": "", + "x": "493", + "y": "202", + } + }, + "planetary_parameters": {}, + } + init_file = tmp_path / "inits.json" + init_file.write_text(json.dumps(init_data)) + + inputs = Inputs(init_opt="y") + inputs.comp_params(init_file, {}) + + assert inputs.info_dict["phot_comp_star"] == {"ra": "", "dec": "", "x": "493", "y": "202"} + + +def test_prereduced_uses_aavso_comp_star_metadata_without_prompt(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#COMP_STAR-XC={\"ra\": null, \"dec\": null, \"x\": \"493\", \"y\": \"202\"}\n" + "#DATE,DIFF,ERR,DETREND_1\n" + "2461102.76092732,0.979108,0.0386426,1.3811172\n" + ) + + inputs = Inputs(init_opt="y") + inputs.info_dict.update({ + "save": str(tmp_path), + "aavso_num": "RTZ", + "second_obs": "", + "date": "2020-01-01", + "lat": "+0.0", + "long": "+0.0", + "elev": 1.0, + "camera": "CCD", + "pixel_bin": "1x1", + "notes": "na", + "aavso_comp": "y", + "prered_file": str(pre_reduced_file), + "exposure": 60.0, + "file_units": "flux", + "file_time": "BJD_TDB", + "phot_comp_star": None, + }) + + info_dict, _ = inputs.prereduced("HAT-P-32 b") + + assert info_dict["phot_comp_star"] == {"ra": "", "dec": "", "x": "493", "y": "202"} + + +def test_prereduced_uses_aavso_observatory_metadata_without_prompt(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#OBSLAT=+32.41638889\n" + "#OBSLON=-110.73444444\n" + "#OBSELEV=2616\n" + "#DATE,DIFF,ERR,DETREND_1\n" + "2461102.76092732,0.979108,0.0386426,1.3811172\n" + ) + + inputs = Inputs(init_opt="y") + inputs.info_dict.update({ + "save": str(tmp_path), + "aavso_num": "RTZ", + "second_obs": "", + "date": "2020-01-01", + "lat": "", + "long": "", + "elev": "", + "camera": "CCD", + "pixel_bin": "1x1", + "notes": "na", + "aavso_comp": "y", + "prered_file": str(pre_reduced_file), + "exposure": 60.0, + "file_units": "flux", + "file_time": "BJD_TDB", + "phot_comp_star": None, + }) + + info_dict, _ = inputs.prereduced("HAT-P-32 b") + + assert info_dict["lat"] == 32.41638889 + assert info_dict["long"] == -110.73444444 + assert info_dict["elev"] == 2616.0 + + +def test_prereduced_uses_aavso_obsdate_metadata_without_prompt(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#OBSDATE=2026-03-08\n" + "#DATE,DIFF,ERR,DETREND_1\n" + "2461102.76092732,0.979108,0.0386426,1.3811172\n" + ) + + inputs = Inputs(init_opt="y") + inputs.info_dict.update({ + "save": str(tmp_path), + "aavso_num": "RTZ", + "second_obs": "", + "date": "", + "lat": "+0.0", + "long": "+0.0", + "elev": 1.0, + "camera": "CCD", + "pixel_bin": "1x1", + "notes": "na", + "aavso_comp": "y", + "prered_file": str(pre_reduced_file), + "exposure": 60.0, + "file_units": "flux", + "file_time": "BJD_TDB", + "phot_comp_star": None, + }) + + info_dict, _ = inputs.prereduced("HAT-P-32 b") + + assert info_dict["date"] == "2026-03-08" + + +def test_prereduced_uses_aavso_filter_and_observing_metadata_without_prompt(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#OBSCODE=\n" + "#SECONDARY_OBSCODES=\n" + "#OBSNAME=Backyard Dome\n" + "#OBSDATE=20260303\n" + "#OBSTYPE=CCD\n" + "#BINNING=1x1\n" + "#EXPOSURE_TIME=30.0\n" + "#OBSLAT=35.554298\n" + "#OBSLON=-105.870197\n" + "#OBSELEV=2194.0\n" + "#GAIADIST=512.4\n" + "#GAIAPMRA=13.25\n" + "#GAIAPMDEC=-7.5\n" + "#NOTES=na\n" + "#DATE_TYPE=BJD_TDB\n" + "#MEASUREMENT_TYPE=Rnflux\n" + "#EXOPLANET_NAME=TOI-1259 A b\n" + "#FILTER=CBB\n" + "#FILTER-XC={\"name\": \"CBB\", \"desc\": \"Astrodon ExoPlanet-BB\", \"fwhm\": [{\"value\": \"500.0\", \"units\": \"nm\"}, {\"value\": \"1000.0\", \"units\": \"nm\"}]}\n" + "#DATE,DIFF,ERR,DETREND_1\n" + "2461102.76092732,0.979108,0.0386426,1.3811172\n" + ) + + inputs = Inputs(init_opt="y") + inputs.info_dict.update({ + "save": str(tmp_path), + "aavso_num": None, + "second_obs": None, + "date": "", + "lat": "", + "long": "", + "elev": "", + "camera": None, + "pixel_bin": None, + "filter": None, + "notes": None, + "aavso_comp": "y", + "prered_file": str(pre_reduced_file), + "exposure": None, + "file_units": None, + "file_time": None, + "phot_comp_star": None, + "wl_min": None, + "wl_max": None, + }) + + info_dict, planet = inputs.prereduced(None) + + assert planet == "TOI-1259 A b" + assert info_dict["aavso_num"] == "" + assert info_dict["second_obs"] == "" + assert info_dict["obs_name"] == "Backyard Dome" + assert info_dict["date"] == "2026-03-03" + assert info_dict["lat"] == 35.554298 + assert info_dict["long"] == -105.870197 + assert info_dict["elev"] == 2194.0 + assert info_dict["camera"] == "CCD" + assert info_dict["pixel_bin"] == "1x1" + assert info_dict["notes"] == "na" + assert info_dict["file_time"] == "BJD_TDB" + assert info_dict["file_units"] == "flux" + assert info_dict["exposure"] == 30.0 + assert info_dict["dist"] == "512.4" + assert info_dict["pm_ra"] == "13.25" + assert info_dict["pm_dec"] == "-7.5" + assert info_dict["filter"] == "CBB" + assert info_dict["wl_min"] == "500.0" + assert info_dict["wl_max"] == "1000.0" + + +def test_parse_aavso_prereduced_overrides_uses_known_filter_lookup_when_filter_xc_missing(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#FILTER=CBB\n" + "#DATE,DIFF,ERR\n" + "2461102.76092732,0.979108,0.0386426\n" + ) + + overrides = parse_aavso_prereduced_overrides(pre_reduced_file) + + assert overrides["filter"] == "CBB" + assert overrides["filter_desc"] == "CBB" + assert overrides["wl_min"] == "500.0" + assert overrides["wl_max"] == "1000.0" + + +def test_parse_aavso_prereduced_overrides_uses_astrodon_exo_alias_lookup(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#FILTER=Astrodon-Exo\n" + "#DATE,DIFF,ERR\n" + "2461102.76092732,0.979108,0.0386426\n" + ) + + overrides = parse_aavso_prereduced_overrides(pre_reduced_file) + + assert overrides["filter"] == "Astrodon-Exo" + assert overrides["filter_desc"] == "CBB" + assert overrides["wl_min"] == "500.0" + assert overrides["wl_max"] == "1000.0" + + +def test_lookup_aavso_filter_metadata_uses_c_alias_for_cv_filter() -> None: + filter_metadata = inputs_module.lookup_aavso_filter_metadata("C") + + assert filter_metadata["name"] == "CV" + assert filter_metadata["desc"] == "CV" + assert filter_metadata["fwhm"] == ("350.0", "1000.0") + + +def test_lookup_aavso_filter_metadata_interprets_cv_as_clearv() -> None: + filter_metadata = inputs_module.lookup_aavso_filter_metadata("CV") + + assert filter_metadata["name"] == "CV" + assert filter_metadata["desc"] == "CV" + assert filter_metadata["fwhm"] == ("350.0", "1000.0") + + +def test_lookup_aavso_filter_metadata_uses_luminosity_aliases_for_clearv_filter() -> None: + for alias in ("lum", "Lum", "Luminosity", "luminosity"): + filter_metadata = inputs_module.lookup_aavso_filter_metadata(alias) + + assert filter_metadata["name"] == "CV" + assert filter_metadata["desc"] == "CV" + assert filter_metadata["fwhm"] == ("350.0", "1000.0") + + +def test_parse_aavso_prereduced_overrides_uses_osc_split_filter_alias_lookup(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#FILTER=G2\n" + "#DATE,DIFF,ERR\n" + "2461102.76092732,0.979108,0.0386426\n" + ) + + overrides = parse_aavso_prereduced_overrides(pre_reduced_file) + + assert overrides["filter"] == "G2" + assert overrides["filter_desc"] == "Photographic G" + assert overrides["wl_min"] == "502.8" + assert overrides["wl_max"] == "586.8" + + +def test_parse_aavso_prereduced_overrides_uses_c_alias_for_cv_filter_lookup(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#FILTER=C\n" + "#DATE,DIFF,ERR\n" + "2461102.76092732,0.979108,0.0386426\n" + ) + + overrides = parse_aavso_prereduced_overrides(pre_reduced_file) + + assert overrides["filter"] == "C" + assert overrides["filter_desc"] == "CV" + assert overrides["wl_min"] == "350.0" + assert overrides["wl_max"] == "1000.0" + + +def test_parse_aavso_prereduced_overrides_uses_luminosity_alias_for_clearv_lookup(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#FILTER=Luminosity\n" + "#DATE,DIFF,ERR\n" + "2461102.76092732,0.979108,0.0386426\n" + ) + + overrides = parse_aavso_prereduced_overrides(pre_reduced_file) + + assert overrides["filter"] == "Luminosity" + assert overrides["filter_desc"] == "CV" + assert overrides["wl_min"] == "350.0" + assert overrides["wl_max"] == "1000.0" + + +def test_parse_aavso_prereduced_overrides_marks_airmass_as_already_corrected(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#DETREND_PARAMETERS=AIRMASS, AIRMASS CORRECTION FUNCTION\n" + "#DATE,DIFF,ERR,DETREND_1,DETREND_2\n" + "2461102.76092732,0.979108,0.0386426,1.3811172,0.998\n" + ) + + overrides = parse_aavso_prereduced_overrides(pre_reduced_file) + + assert overrides["airmass_already_corrected"] is True + + +def test_prereduced_prefers_aavso_obsdate_metadata_over_init_date(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#OBSDATE=2026-03-08\n" + "#DATE,DIFF,ERR,DETREND_1\n" + "2461102.76092732,0.979108,0.0386426,1.3811172\n" + ) + + inputs = Inputs(init_opt="y") + inputs.info_dict.update({ + "save": str(tmp_path), + "aavso_num": "RTZ", + "second_obs": "", + "date": "1999-01-01", + "lat": "+0.0", + "long": "+0.0", + "elev": 1.0, + "camera": "CCD", + "pixel_bin": "1x1", + "notes": "na", + "aavso_comp": "y", + "prered_file": str(pre_reduced_file), + "exposure": 60.0, + "file_units": "flux", + "file_time": "BJD_TDB", + "phot_comp_star": None, + }) + + info_dict, _ = inputs.prereduced("HAT-P-32 b") + + assert info_dict["date"] == "2026-03-08" + + +def test_prereduced_derives_obsdate_from_first_data_row_without_prompt(tmp_path): + pre_reduced_file = tmp_path / "prereduced.txt" + pre_reduced_file.write_text( + "time,flux,uncertainty\n" + "2458849.5,0.979108,0.0386426\n" + ) + + inputs = Inputs(init_opt="y") + inputs.info_dict.update({ + "save": str(tmp_path), + "aavso_num": "RTZ", + "second_obs": "", + "date": "", + "lat": "+0.0", + "long": "+0.0", + "elev": 1.0, + "camera": "CCD", + "pixel_bin": "1x1", + "notes": "na", + "aavso_comp": "y", + "prered_file": str(pre_reduced_file), + "exposure": 60.0, + "file_units": "flux", + "file_time": "JD_UTC", + "phot_comp_star": None, + }) + + info_dict, _ = inputs.prereduced("HAT-P-32 b") + + assert info_dict["date"] == "2020-01-01" + + +def test_prereduced_leaves_phot_comp_star_blank_when_missing_from_aavso_metadata(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#DATE,DIFF,ERR,DETREND_1\n" + "2461102.76092732,0.979108,0.0386426,1.3811172\n" + ) + + inputs = Inputs(init_opt="y") + inputs.info_dict.update({ + "save": str(tmp_path), + "aavso_num": "RTZ", + "second_obs": "", + "date": "2020-01-01", + "lat": "+0.0", + "long": "+0.0", + "elev": 1.0, + "camera": "CCD", + "pixel_bin": "1x1", + "notes": "na", + "aavso_comp": "y", + "prered_file": str(pre_reduced_file), + "exposure": 60.0, + "file_units": "flux", + "file_time": "BJD_TDB", + "phot_comp_star": None, + }) + + info_dict, _ = inputs.prereduced("HAT-P-32 b") + + assert info_dict["phot_comp_star"] == {"ra": "", "dec": "", "x": "", "y": ""} + + +def test_prereduced_carries_aavso_airmass_corrected_flag(tmp_path): + pre_reduced_file = tmp_path / "aavso_prereduced.txt" + pre_reduced_file.write_text( + "#TYPE=EXOPLANET\n" + "#DETREND_PARAMETERS=AIRMASS, AIRMASS CORRECTION FUNCTION\n" + "#DATE,DIFF,ERR,DETREND_1,DETREND_2\n" + "2461102.76092732,0.979108,0.0386426,1.3811172,0.998\n" + ) + + inputs = Inputs(init_opt="y") + inputs.info_dict.update({ + "save": str(tmp_path), + "aavso_num": "RTZ", + "second_obs": "", + "date": "2020-01-01", + "lat": "+0.0", + "long": "+0.0", + "elev": 1.0, + "camera": "CCD", + "pixel_bin": "1x1", + "notes": "na", + "aavso_comp": "y", + "prered_file": str(pre_reduced_file), + "exposure": 60.0, + "file_units": "flux", + "file_time": "BJD_TDB", + "phot_comp_star": None, + }) + + info_dict, _ = inputs.prereduced("HAT-P-32 b") + + assert info_dict["airmass_already_corrected"] is True diff --git a/tests/test_lazy_pylightcurve_imports.py b/tests/test_lazy_pylightcurve_imports.py new file mode 100644 index 00000000..46e01785 --- /dev/null +++ b/tests/test_lazy_pylightcurve_imports.py @@ -0,0 +1,277 @@ +import subprocess +import sys +import textwrap +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def test_imports_eagerly_load_pylightcurve_without_noise(): + script = textwrap.dedent( + """ + import tempfile + import sys + import types + from pathlib import Path + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + package_dir = root / "pylightcurve" + model_dir = package_dir / "models" + model_dir.mkdir(parents=True) + + (package_dir / "__init__.py").write_text("", encoding="utf-8") + (model_dir / "__init__.py").write_text("", encoding="utf-8") + (model_dir / "exoplanet_lc.py").write_text( + "import sys\\n" + "print('LOUD-STDOUT')\\n" + "print('LOUD-STDERR', file=sys.stderr)\\n" + "def transit(*args, **kwargs): return 'stub-transit'\\n" + "def eclipse_mid_time(*args, **kwargs): return 0.0\\n", + encoding="utf-8", + ) + + sys.path.insert(0, str(root)) + + for name in list(sys.modules): + if name == "exotic.api.elca" or name == "exotic.api.joint_fitter" or name.startswith("pylightcurve"): + sys.modules.pop(name) + + fake_ultranest = types.ModuleType("ultranest") + fake_ultranest.ReactiveNestedSampler = type("ReactiveNestedSampler", (), {}) + sys.modules["ultranest"] = fake_ultranest + fake_plotting = types.ModuleType("plotting") + fake_plotting.corner = lambda *args, **kwargs: None + sys.modules["plotting"] = fake_plotting + sys.modules["exotic.api.plotting"] = fake_plotting + fake_ultranest_utils = types.ModuleType("ultranest_utils") + fake_ultranest_utils.run_reactive_sampler = lambda *args, **kwargs: None + sys.modules["ultranest_utils"] = fake_ultranest_utils + sys.modules["exotic.api.ultranest_utils"] = fake_ultranest_utils + + import exotic.api.elca as elca + import exotic.api.joint_fitter as joint_fitter + + assert "pylightcurve.models.exoplanet_lc" in sys.modules + + minimal_values = { + "u0": 0.0, + "u1": 0.0, + "u2": 0.0, + "u3": 0.0, + "rprs": 0.1, + "per": 1.0, + "ars": 10.0, + "ecc": 0.0, + "inc": 89.0, + "omega": 90.0, + "tmid": 0.0, + } + + assert elca.transit([0.0], minimal_values) == "stub-transit" + assert joint_fitter.pytransit([0.0, 0.0, 0.0, 0.0], 0.1, 1.0, 10.0, 0.0, 89.0, 90.0, 0.0, [0.0]) == "stub-transit" + print("imports-ok") + """ + ) + + result = subprocess.run( + [sys.executable, "-c", script], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr or result.stdout + assert "imports-ok" in result.stdout + assert "Importing modules. Please wait......." not in result.stdout + assert "LOUD-STDOUT" not in result.stdout + assert "LOUD-STDERR" not in result.stderr + + +def test_transit_supersamples_points_with_significant_exposure_smearing(): + script = textwrap.dedent( + """ + import sys + import tempfile + import types + from pathlib import Path + + import numpy as np + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + package_dir = root / "pylightcurve" + model_dir = package_dir / "models" + model_dir.mkdir(parents=True) + + (package_dir / "__init__.py").write_text("", encoding="utf-8") + (model_dir / "__init__.py").write_text("", encoding="utf-8") + (model_dir / "exoplanet_lc.py").write_text( + "import numpy as np\\n" + "def transit(*args, **kwargs):\\n" + " times = np.asarray(args[-1], dtype=float)\\n" + " return 1.0 + times ** 2\\n" + "def eclipse_mid_time(*args, **kwargs): return 0.0\\n", + encoding="utf-8", + ) + + sys.path.insert(0, str(root)) + for name in list(sys.modules): + if name == "exotic.api.elca" or name.startswith("pylightcurve"): + sys.modules.pop(name) + + fake_ultranest = types.ModuleType("ultranest") + fake_ultranest.ReactiveNestedSampler = type("ReactiveNestedSampler", (), {}) + sys.modules["ultranest"] = fake_ultranest + fake_plotting = types.ModuleType("plotting") + fake_plotting.corner = lambda *args, **kwargs: None + sys.modules["plotting"] = fake_plotting + sys.modules["exotic.api.plotting"] = fake_plotting + fake_ultranest_utils = types.ModuleType("ultranest_utils") + fake_ultranest_utils.run_reactive_sampler = lambda *args, **kwargs: None + sys.modules["ultranest_utils"] = fake_ultranest_utils + sys.modules["exotic.api.ultranest_utils"] = fake_ultranest_utils + + import exotic.api.elca as elca + + values = { + "u0": 0.0, + "u1": 0.0, + "u2": 0.0, + "u3": 0.0, + "rprs": 0.1, + "per": 1.0, + "ars": 10.0, + "ecc": 0.0, + "inc": 90.0, + "omega": 90.0, + "tmid": 0.0, + elca.EXPOSURE_SMEARING_EXPOSURE_TIME_KEY: np.array([3600.0 / 86400.0]), + elca.EXPOSURE_SMEARING_SUPERSAMPLE_KEY: 5, + elca.EXPOSURE_SMEARING_CHANGE_TOLERANCE_KEY: 0.0, + } + + result = elca.transit(np.array([0.0]), values) + offsets = (np.arange(5, dtype=float) + 0.5) / 5.0 - 0.5 + expected = np.mean(1.0 + ((3600.0 / 86400.0) * offsets) ** 2) + assert np.allclose(result, [expected]) + assert result[0] > 1.0 + print("smearing-ok") + """ + ) + + result = subprocess.run( + [sys.executable, "-c", script], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr or result.stdout + assert "smearing-ok" in result.stdout + + +def test_import_exotic_avoids_unused_astroquery_modules(): + script = textwrap.dedent( + """ + import sys + import tempfile + import types + from pathlib import Path + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + astroquery_dir = root / "astroquery" + astroquery_dir.mkdir(parents=True) + (astroquery_dir / "__init__.py").write_text("", encoding="utf-8") + (astroquery_dir / "simbad.py").write_text( + "import sys\\n" + "print('LOUD-SIMBAD-STDOUT')\\n" + "print('LOUD-SIMBAD-STDERR', file=sys.stderr)\\n" + "class Simbad:\\n pass\\n", + encoding="utf-8", + ) + (astroquery_dir / "gaia.py").write_text( + "import sys\\n" + "print('LOUD-GAIA-STDOUT')\\n" + "print('LOUD-GAIA-STDERR', file=sys.stderr)\\n" + "class Gaia:\\n pass\\n", + encoding="utf-8", + ) + sys.path.insert(0, str(root)) + + fake_barycorrpy = types.ModuleType("barycorrpy") + fake_utc_tdb = types.ModuleType("barycorrpy.utc_tdb") + fake_utc_tdb.JDUTC_to_BJDTDB = lambda *args, **kwargs: None + fake_astroalign = types.ModuleType("astroalign") + fake_astroalign.PIXEL_TOL = 1 + fake_imreg_dft = types.ModuleType("imreg_dft") + fake_colour_demosaicing = types.ModuleType("colour_demosaicing") + fake_colour_demosaicing.demosaicing_CFA_Bayer_bilinear = lambda *args, **kwargs: None + fake_photutils = types.ModuleType("photutils") + fake_photutils_aperture = types.ModuleType("photutils.aperture") + fake_photutils_aperture.CircularAperture = type("CircularAperture", (), {}) + fake_photutils_aperture.CircularAnnulus = type("CircularAnnulus", (), {}) + fake_photutils_detection = types.ModuleType("photutils.detection") + fake_photutils_detection.DAOStarFinder = type("DAOStarFinder", (), {}) + fake_ldtk = types.ModuleType("ldtk") + fake_ldtk.LDPSet = type("LDPSet", (), {}) + fake_ldtk.ldtk = types.SimpleNamespace(LDPSet=fake_ldtk.LDPSet) + fake_ldtk_ldmodel = types.ModuleType("ldtk.ldmodel") + fake_ldtk_ldmodel.LinearModel = type("LinearModel", (), {}) + fake_ldtk_ldmodel.QuadraticModel = type("QuadraticModel", (), {}) + fake_ldtk_ldmodel.NonlinearModel = type("NonlinearModel", (), {}) + fake_lmfit = types.ModuleType("lmfit") + fake_pyvo = types.ModuleType("pyvo") + fake_ultranest = types.ModuleType("ultranest") + fake_ultranest.ReactiveNestedSampler = type("ReactiveNestedSampler", (), {}) + fake_elca = types.ModuleType("exotic.api.elca") + fake_elca.lc_fitter = lambda *args, **kwargs: None + fake_elca.binner = lambda *args, **kwargs: None + fake_elca.transit = lambda *args, **kwargs: None + fake_elca.get_phase = lambda *args, **kwargs: None + fake_ld = types.ModuleType("exotic.api.ld") + fake_ld.LimbDarkening = type("LimbDarkening", (), {}) + fake_ld.ld_re_punct_p = lambda *args, **kwargs: None + + sys.modules.setdefault("astroalign", fake_astroalign) + sys.modules.setdefault("barycorrpy", fake_barycorrpy) + sys.modules.setdefault("barycorrpy.utc_tdb", fake_utc_tdb) + sys.modules.setdefault("imreg_dft", fake_imreg_dft) + sys.modules.setdefault("colour_demosaicing", fake_colour_demosaicing) + sys.modules.setdefault("photutils", fake_photutils) + sys.modules.setdefault("photutils.aperture", fake_photutils_aperture) + sys.modules.setdefault("photutils.detection", fake_photutils_detection) + sys.modules.setdefault("ldtk", fake_ldtk) + sys.modules.setdefault("ldtk.ldmodel", fake_ldtk_ldmodel) + sys.modules.setdefault("lmfit", fake_lmfit) + sys.modules.setdefault("pyvo", fake_pyvo) + sys.modules.setdefault("ultranest", fake_ultranest) + sys.modules.setdefault("exotic.api.elca", fake_elca) + sys.modules.setdefault("exotic.api.ld", fake_ld) + + import exotic.exotic + + assert "astroquery.gaia" not in sys.modules + assert "astroquery.simbad" not in sys.modules + print("exotic-import-ok") + """ + ) + + result = subprocess.run( + [sys.executable, "-c", script], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr or result.stdout + assert "exotic-import-ok" in result.stdout + assert "LOUD-SIMBAD-STDOUT" not in result.stdout + assert "LOUD-SIMBAD-STDERR" not in result.stderr + assert "LOUD-GAIA-STDOUT" not in result.stdout + assert "LOUD-GAIA-STDERR" not in result.stderr diff --git a/tests/test_ld.py b/tests/test_ld.py index 794b9c94..fd0ab1c2 100644 --- a/tests/test_ld.py +++ b/tests/test_ld.py @@ -1,3 +1,5 @@ +import logging + from exotic.api.ld import LimbDarkening stellar_params = { @@ -73,7 +75,7 @@ def test_existing_standard_filter_alias_name() -> None: assert observed_filter == expected_filter -def test_existing_mobs_standard_filter_name() -> None: +def test_legacy_mobs_filter_name_maps_to_cv() -> None: observed_filter = { 'filter': "MObs CV", 'name': None, @@ -82,17 +84,17 @@ def test_existing_mobs_standard_filter_name() -> None: } expected_filter = { - 'filter': "MObs CV", + 'filter': "CV", 'name': 'CV', 'wl_min': '350.0', - 'wl_max': '850.0' + 'wl_max': '1000.0' } setting_filter_values(observed_filter) assert observed_filter == expected_filter -def test_custom_nonspecific_standard_filter_abbreviation_1() -> None: +def test_cv_standard_filter_abbreviation_uses_clearv() -> None: observed_filter = { 'filter': "CV", 'name': None, @@ -102,7 +104,13 @@ def test_custom_nonspecific_standard_filter_abbreviation_1() -> None: ld_obj = LimbDarkening(stellar_params) - assert ld_obj.check_standard(observed_filter) == False + assert ld_obj.check_standard(observed_filter) is True + assert observed_filter == { + 'filter': "CV", + 'name': "CV", + 'wl_min': "350.0", + 'wl_max': "1000.0", + } def test_custom_nonspecific_standard_filter_abbreviation_2() -> None: observed_filter = { @@ -147,19 +155,19 @@ def test_existing_standard_filter_fwhm() -> None: assert observed_filter == expected_filter -def test_existing_mobs_standard_filter_mobs() -> None: +def test_existing_clearv_standard_filter_wavelengths() -> None: observed_filter = { 'filter': None, 'name': None, 'wl_min': '350.0', - 'wl_max': '850.0' + 'wl_max': '1000.0' } expected_filter = { - 'filter': "MObs CV", + 'filter': "CV", 'name': 'CV', 'wl_min': '350.0', - 'wl_max': '850.0' + 'wl_max': '1000.0' } setting_filter_values(observed_filter) @@ -190,6 +198,21 @@ def test_valid_fwhm_range_swapped_min_max() -> None: assert ld_obj.check_fwhm(observed_filter) == True +def test_missing_fwhm_values_do_not_log_errors(caplog) -> None: + observed_filter = { + 'filter': None, + 'name': None, + 'wl_min': None, + 'wl_max': None + } + + ld_obj = LimbDarkening(stellar_params) + + with caplog.at_level(logging.ERROR, logger="exotic.api.ld"): + assert ld_obj.check_fwhm(observed_filter) == False + + assert "FWHM matching failed" not in caplog.text + def test_invalid_fwhm_range_1() -> None: observed_filter = { 'filter': None, @@ -213,3 +236,104 @@ def test_invalid_fwhm_range_2() -> None: ld_obj = LimbDarkening(stellar_params) assert ld_obj.check_fwhm(observed_filter) == False + + +def test_photographic_filter_aliases_in_filter_column() -> None: + alias_cases = [ + ("pb", "Photographic B", "PB", "391.6", "480.6"), + ("G", "Photographic G", "PG", "502.8", "586.8"), + ("pg", "Photographic G", "PG", "502.8", "586.8"), + ("pr", "Photographic R", "PR", "590.0", "810.0"), + ] + + for alias, expected_filter, expected_name, expected_min, expected_max in alias_cases: + observed_filter = {'filter': alias, 'name': None, 'wl_min': None, 'wl_max': None} + setting_filter_values(observed_filter) + assert observed_filter == { + 'filter': expected_filter, + 'name': expected_name, + 'wl_min': expected_min, + 'wl_max': expected_max, + } + + +def test_cbb_filter_uses_neutral_canonical_name() -> None: + observed_filter = {'filter': "CBB", 'name': None, 'wl_min': None, 'wl_max': None} + + setting_filter_values(observed_filter) + + assert observed_filter == { + 'filter': "CBB", + 'name': "CBB", + 'wl_min': "500.0", + 'wl_max': "1000.0", + } + + +def test_cbb_brand_names_remain_accepted_as_input_aliases() -> None: + for alias in ("Astrodon ExoPlanet-BB", "Astrodon-Exo", "Exop", "exo"): + observed_filter = {'filter': alias, 'name': None, 'wl_min': None, 'wl_max': None} + + setting_filter_values(observed_filter) + + assert observed_filter == { + 'filter': "CBB", + 'name': "CBB", + 'wl_min': "500.0", + 'wl_max': "1000.0", + } + + +def test_additional_standard_filter_aliases_in_filter_column() -> None: + alias_cases = [ + ("bu", "Johnson U", "U", "333.8", "398.8"), + ("bi", "Johnson I", "IJ", "780.0", "1020.0"), + ("up", "Sloan u", "SU", "321.8", "386.8"), + ("gp", "Sloan g", "SG", "402.5", "551.5"), + ("rp", "Sloan r", "SR", "553.1", "693.1"), + ("ip", "Sloan i", "SI", "697.5", "827.5"), + ("zp", "Sloan z", "SZ", "841.2", "978.2"), + ("su", "Stromgren u", "STU", "336.3", "367.7"), + ("sv", "Stromgren v", "STV", "401.5", "418.5"), + ("sb", "Stromgren b", "STB", "459.55", "478.05"), + ("sy", "Stromgren y", "STY", "536.7", "559.3"), + ("hb", "Stromgren Hbw", "STHBW", "481.5", "496.5"), + ("zs", "PanSTARRS z-short", "ZS", "826.0", "920.0"), + ("CV", "CV", "CV", "350.0", "1000.0"), + ("clearV", "CV", "CV", "350.0", "1000.0"), + ("w", "CV", "CV", "350.0", "1000.0"), + ("pl", "CV", "CV", "350.0", "1000.0"), + ("exo", "CBB", "CBB", "500.0", "1000.0"), + ("Astrodon ExoPlanet-BB", "CBB", "CBB", "500.0", "1000.0"), + ("Astrodon-Exo", "CBB", "CBB", "500.0", "1000.0"), + ] + + for alias, expected_filter, expected_name, expected_min, expected_max in alias_cases: + observed_filter = {'filter': alias, 'name': None, 'wl_min': None, 'wl_max': None} + setting_filter_values(observed_filter) + assert observed_filter == { + 'filter': expected_filter, + 'name': expected_name, + 'wl_min': expected_min, + 'wl_max': expected_max, + } + + +def test_osc_split_filter_aliases_in_filter_column() -> None: + alias_cases = [ + ("B1", "Photographic B", "PB", "391.6", "480.6"), + ("G1", "Photographic G", "PG", "502.8", "586.8"), + ("G2", "Photographic G", "PG", "502.8", "586.8"), + ("R1", "Photographic R", "PR", "590.0", "810.0"), + ("R2", "Photographic R", "PR", "590.0", "810.0"), + ] + + for alias, expected_filter, expected_name, expected_min, expected_max in alias_cases: + observed_filter = {'filter': alias, 'name': None, 'wl_min': None, 'wl_max': None} + setting_filter_values(observed_filter) + assert observed_filter == { + 'filter': expected_filter, + 'name': expected_name, + 'wl_min': expected_min, + 'wl_max': expected_max, + } diff --git a/tests/test_ldtk_http_fallback.py b/tests/test_ldtk_http_fallback.py new file mode 100644 index 00000000..25dd4745 --- /dev/null +++ b/tests/test_ldtk_http_fallback.py @@ -0,0 +1,172 @@ +from pathlib import Path + +import pytest + +pytest.importorskip("ldtk") + +from exotic.api import gael_ld # noqa: E402 + + +class DummyResponse: + def __init__(self, chunks=None, text=""): + self._chunks = chunks + self.text = text + self.closed = False + + def raise_for_status(self): + return None + + def iter_content(self, chunk_size): + return iter(self._chunks) + + def close(self): + self.closed = True + + +class DummyLDTkFile: + def __init__(self, cache_path): + self.name = "lte02300+0.00+0.5.PHOENIX-ACES-AGSS-COND-SPECINT-2011.fits" + self._zstr = "Z+0.5" + self.local_path = str(Path(cache_path) / self._zstr / self.name) + + @property + def local_exists(self): + return Path(self.local_path).exists() + + +class DummyClient: + def __init__(self, cache_path): + self.edir = "SpecInt50FITS/PHOENIX-ACES-AGSS-COND-SPECINT-2011" + self.files = [DummyLDTkFile(cache_path)] + self.not_cached = len(self.files) + self.checked_paths = None + + def check_file_corruption(self, paths): + self.checked_paths = paths + return False + + +def test_ldtk_http_fallback_downloads_missing_file_to_ldtk_cache(monkeypatch, tmp_path): + client = DummyClient(tmp_path / "cache_vis-lowres") + captured = {} + + def fake_get(url, stream, timeout): + captured["url"] = url + captured["stream"] = stream + captured["timeout"] = timeout + return DummyResponse([b"phoenix", b"", b"-fits"]) + + monkeypatch.setenv(gael_ld._LDTK_HTTP_FALLBACK_ENV, "https://mirror.example/PHOENIX/") + monkeypatch.setattr(gael_ld.requests, "get", fake_get) + + assert gael_ld._download_ldtk_uncached_files_from_http(client) is False + + assert captured == { + "url": ( + "https://mirror.example/PHOENIX/" + "SpecInt50FITS/PHOENIX-ACES-AGSS-COND-SPECINT-2011/" + "Z%2B0.5/lte02300%2B0.00%2B0.5.PHOENIX-ACES-AGSS-COND-SPECINT-2011.fits" + ), + "stream": True, + "timeout": gael_ld._LDTK_DOWNLOAD_TIMEOUT, + } + assert Path(client.files[0].local_path).read_bytes() == b"phoenix-fits" + assert client.checked_paths == [client.files[0].local_path] + assert client.not_cached == 0 + + +def test_ldtk_http_fallback_tries_gwdg_before_nextastro(monkeypatch, tmp_path): + client = DummyClient(tmp_path / "cache_vis-lowres") + requested_urls = [] + + def fake_get(url, stream, timeout): + requested_urls.append(url) + if url.startswith("https://ftp.gwdg.de/"): + raise RuntimeError("gwdg unavailable") + return DummyResponse([b"nextastro"]) + + monkeypatch.delenv(gael_ld._LDTK_HTTP_FALLBACK_ENV, raising=False) + monkeypatch.setattr(gael_ld.requests, "get", fake_get) + + assert gael_ld._download_ldtk_uncached_files_from_http(client) is False + + assert requested_urls[0].startswith("https://ftp.gwdg.de/pub/misc/phoenix/") + assert requested_urls[1].startswith("https://downloads.nextastro.org/PHOENIX/") + assert Path(client.files[0].local_path).read_bytes() == b"nextastro" + + +def test_ldtk_download_wrapper_tries_original_before_http_fallback(monkeypatch, tmp_path): + from ldtk.client import Client + + client = DummyClient(tmp_path / "cache_vis-lowres") + calls = [] + + def fake_original(self, force=False): + calls.append(("ftp", force)) + raise RuntimeError("ftp unavailable") + + def fake_fallback(self, force=False): + calls.append(("http", force)) + return False + + monkeypatch.setattr(gael_ld, "_LDTK_ORIGINAL_DOWNLOAD_UNCACHED_FILES", fake_original) + monkeypatch.setattr(gael_ld, "_download_ldtk_uncached_files_from_http", fake_fallback) + + assert Client.download_uncached_files(client, force=True) is False + assert calls == [("ftp", True), ("http", True)] + + +def test_ldtk_file_list_wrapper_uses_http_fallback_when_ftp_listing_fails(monkeypatch): + from ldtk.client import Client + + class ClientStub: + edir = "SpecInt50FITS/PHOENIX-ACES-AGSS-COND-SPECINT-2011" + + calls = [] + requested_urls = [] + + def fake_original(self): + calls.append("ftp") + raise TimeoutError("ftp listing timed out") + + def fake_get(url, timeout): + requested_urls.append(url) + if url.endswith("PHOENIX-ACES-AGSS-COND-SPECINT-2011/"): + return DummyResponse(text=""" + ../ + README.txt + Z+0.5/ + Z-0.0/ + """) + if url.endswith("Z%2B0.5/"): + return DummyResponse(text=""" + ../ + + lte02300+0.00+0.5.PHOENIX-ACES-AGSS-COND-SPECINT-2011.fits + + """) + if url.endswith("Z-0.0/"): + return DummyResponse(text=""" + ../ + + lte02300-0.00-0.0.PHOENIX-ACES-AGSS-COND-SPECINT-2011.fits + + """) + raise AssertionError(f"unexpected URL: {url}") + + monkeypatch.setenv(gael_ld._LDTK_HTTP_FALLBACK_ENV, "https://mirror.example/PHOENIX") + monkeypatch.setattr(gael_ld, "_LDTK_ORIGINAL_GET_SERVER_FILE_LIST", fake_original) + monkeypatch.setattr(gael_ld.requests, "get", fake_get) + + files = Client.get_server_file_list(ClientStub()) + + assert calls == ["ftp"] + assert requested_urls == [ + "https://mirror.example/PHOENIX/SpecInt50FITS/PHOENIX-ACES-AGSS-COND-SPECINT-2011/", + "https://mirror.example/PHOENIX/SpecInt50FITS/PHOENIX-ACES-AGSS-COND-SPECINT-2011/Z%2B0.5/", + "https://mirror.example/PHOENIX/SpecInt50FITS/PHOENIX-ACES-AGSS-COND-SPECINT-2011/Z-0.0/", + ] + assert files == { + "Z+0.5": ["lte02300+0.00+0.5.PHOENIX-ACES-AGSS-COND-SPECINT-2011.fits"], + "Z-0.0": ["lte02300-0.00-0.0.PHOENIX-ACES-AGSS-COND-SPECINT-2011.fits"], + } diff --git a/tests/test_nea_nextastro_fallback.py b/tests/test_nea_nextastro_fallback.py new file mode 100644 index 00000000..2b413928 --- /dev/null +++ b/tests/test_nea_nextastro_fallback.py @@ -0,0 +1,153 @@ +import pandas +import pytest +import requests + +from exotic.api.nea import NASAExoplanetArchive, planet_name_lookup_candidates + + +class DummyResponse: + def __init__(self, payload): + self._payload = payload + + def raise_for_status(self): + return None + + def json(self): + return self._payload + + +def test_planet_info_uses_nextastro_fallback_when_nasa_archive_unavailable(monkeypatch): + nea = NASAExoplanetArchive('WASP-12 b') + + def raise_direct_failure(*args, **kwargs): + raise requests.exceptions.RequestException('ipac unavailable') + + monkeypatch.setattr(nea, '_new_scrape', raise_direct_failure) + + payload = { + 'params': { + 'name': 'WASP-12 b', + 'hostStarName': 'WASP-12', + 'raDeg': 180.0, + 'decDeg': 29.0, + 'orbitalPeriodDays': {'value': 1.09, 'errPlus': 0.001, 'errMinus': 0.001}, + 'midTransitTimeDays': {'value': 2450000.5, 'errPlus': 0.0002, 'errMinus': 0.0003}, + 'rpOverRs': {'value': 0.12, 'errPlus': 0.004, 'errMinus': 0.003}, + 'aOverRs': {'value': 3.0, 'errPlus': 0.2, 'errMinus': 0.1}, + 'inclinationDeg': {'value': 83.5, 'errPlus': 0.8, 'errMinus': 0.7}, + 'eccentricity': 0.0, + 'argPeriastronDeg': 0.0, + 'starTeffK': {'value': 6300.0, 'errPlus': 50.0, 'errMinus': 40.0}, + 'starFeh': {'value': 0.2, 'errPlus': 0.03, 'errMinus': 0.02}, + 'starLogg': {'value': 4.1, 'errPlus': 0.05, 'errMinus': 0.04}, + 'source': {'table': 'pscomppars', 'localCache': True}, + } + } + + called = {} + + def fake_get(url, params, timeout): + called['url'] = url + called['params'] = params + called['timeout'] = timeout + return DummyResponse(payload) + + monkeypatch.setattr(requests, 'get', fake_get) + + planet_name, candidate, pl_dict = nea.planet_info() + + assert planet_name == 'WASP-12 b' + assert candidate is False + assert called['url'] == 'https://archive.nextastro.org/api/exoplanet_params' + assert called['params'] == {'name': 'WASP-12 b'} + assert pl_dict['pName'] == 'WASP-12 b' + assert pl_dict['sName'] == 'WASP-12' + assert pl_dict['ra'] == 180.0 + assert pl_dict['dec'] == 29.0 + assert pl_dict['pPer'] == 1.09 + assert pl_dict['midT'] == 2450000.5 + assert pl_dict['rprs'] == 0.12 + assert pl_dict['aRs'] == 3.0 + + +@pytest.mark.parametrize( + ("planet_name", "reason"), + [ + ("TOI-3889.01", "the name ends with a decimal suffix"), + ("TIC 123456789", "the name starts with 'TIC'"), + ], +) +def test_new_scrape_auto_marks_candidate_like_names_without_prompt(monkeypatch, tmp_path, capsys, + planet_name, reason): + nea = NASAExoplanetArchive(planet_name) + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(nea, 'planet_names', lambda filename="pl_names.json": None) + monkeypatch.setattr(nea, '_tap_query', lambda *args, **kwargs: pandas.DataFrame()) + monkeypatch.setattr( + 'builtins.input', + lambda prompt: pytest.fail("interactive prompt should not run for candidate-like targets"), + ) + + resolved_name, candidate = nea._new_scrape() + + assert resolved_name == planet_name + assert candidate is True + + output = capsys.readouterr().out + assert f"Cannot find target ({planet_name}) in NASA Exoplanet Archive." in output + assert f"Assuming {planet_name} is a planet candidate because {reason}." in output + + +def test_new_scrape_non_interactive_unknown_target_aborts_without_prompt(monkeypatch, tmp_path): + planet_name = 'Definitely Not A Planet b' + nea = NASAExoplanetArchive(planet_name, non_interactive=True) + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(nea, 'planet_names', lambda filename="pl_names.json": None) + monkeypatch.setattr(nea, '_tap_query', lambda *args, **kwargs: pandas.DataFrame()) + monkeypatch.setattr( + 'builtins.input', + lambda prompt: pytest.fail("non-interactive NASA lookup must not prompt"), + ) + + with pytest.raises( + RuntimeError, + match=r"Non-interactive run cancelled: target \(Definitely Not A Planet b\) was not found", + ): + nea._new_scrape() + + +def test_planet_name_lookup_candidates_strip_phase_and_preserve_planet_letter(): + candidates = planet_name_lookup_candidates('field WASP-164 b ingress') + + assert 'field WASP-164 b' in candidates + assert 'field WASP-164b' in candidates + assert 'WASP-164b' in candidates + assert 'ingress' not in candidates + + +@pytest.mark.parametrize('phase', ['ingress', 'EGRESS']) +def test_new_scrape_resolves_phase_labeled_name_from_planet_cache( + monkeypatch, + tmp_path, + phase, +): + nea = NASAExoplanetArchive(f'WASP-164 b {phase}', non_interactive=True) + monkeypatch.chdir(tmp_path) + (tmp_path / 'pl_names.json').write_text( + '{"wasp164b": "WASP-164 b"}', + encoding='utf-8', + ) + + class LookupResolved(Exception): + pass + + def stop_after_name_resolution(*args, **kwargs): + assert nea.planet == 'WASP-164 b' + raise LookupResolved + + monkeypatch.setattr(nea, '_tap_query', stop_after_name_resolution) + + with pytest.raises(LookupResolved): + nea._new_scrape() diff --git a/tests/test_nextastro_astrometry.py b/tests/test_nextastro_astrometry.py new file mode 100644 index 00000000..3ea5afa4 --- /dev/null +++ b/tests/test_nextastro_astrometry.py @@ -0,0 +1,287 @@ +import gzip +import json +from pathlib import Path + +import numpy as np +from astropy.io.fits import getheader, writeto +import pytest + +from exotic.api.plate_solution import NextAstroPlateSolution, PlateSolution + + +class DummyResponse: + def __init__(self, payload=None, status_code=200, text=None, json_error=None): + self._payload = payload + self.status_code = status_code + self.text = text if text is not None else ("" if payload is None else str(payload)) + self._json_error = json_error + + def json(self): + if self._json_error is not None: + raise self._json_error + return self._payload + + +def _decode_request_body(body, headers): + encoding = headers["Content-Encoding"] + if encoding == "gzip": + return json.loads(gzip.decompress(body).decode("utf-8")) + if encoding == "zstd": + zstandard = pytest.importorskip("zstandard") + return json.loads(zstandard.ZstdDecompressor().decompress(body).decode("utf-8")) + raise AssertionError(f"Unexpected content encoding: {encoding}") + + +def _create_test_fits(tmp_path: Path) -> Path: + image = np.zeros((100, 120), dtype=float) + image[30, 25] = 10000.0 + image[70, 80] = 8000.0 + image[50, 60] = 7000.0 + fits_path = tmp_path / "image.fits" + writeto(fits_path, image, overwrite=True) + return fits_path + + +def test_generate_source_list(tmp_path): + fits_path = _create_test_fits(tmp_path) + solver = NextAstroPlateSolution(file=fits_path, directory=tmp_path) + + source_list = solver._generate_source_list() + + assert source_list is not None + assert source_list["pixel_indexing"] == "0-based" + assert source_list["origin"] == "exotic" + assert len(source_list["x"]) > 0 + assert len(source_list["x"]) == len(source_list["y"]) == len(source_list["flux"]) + + +def test_plate_solution_writes_wcs_file(tmp_path, monkeypatch): + fits_path = _create_test_fits(tmp_path) + (tmp_path / "working_artifacts").mkdir() + + def fake_post(url, data, headers, timeout): + payload = _decode_request_body(data, headers) + assert url.endswith('/solve') + assert headers["Content-Type"] == "application/json" + assert headers["Content-Encoding"] in {"gzip", "zstd"} + assert headers["X-NextAstro-Software"].startswith("EXOTIC/") + assert payload["sources"]["origin"] == "exotic" + assert payload['image'] == {'width': 120, 'height': 100} + assert payload['hints']['ra_deg'] == 210.8023 + assert payload['hints']['dec_deg'] == 54.3489 + assert payload['hints']['scale_arcsec_per_pix'] == 1.23 + assert payload['hints']['scale_tolerance_frac'] == 0.25 + return DummyResponse({'status': 'queued', 'request_id': 'abc123'}) + + def fake_get(url, timeout): + assert url.endswith('/status/abc123') + return DummyResponse({ + 'status': 'solved', + 'solution': { + 'wcs_header': { + 'SIMPLE': True, + 'BITPIX': -64, + 'NAXIS': 2, + 'NAXIS1': 120, + 'NAXIS2': 100, + 'CTYPE1': 'RA---TAN', + 'CTYPE2': 'DEC--TAN', + 'CRVAL1': 210.8, + 'CRVAL2': 54.3, + 'CRPIX1': 60.0, + 'CRPIX2': 50.0, + 'CD1_1': -0.00028, + 'CD1_2': 0.0, + 'CD2_1': 0.0, + 'CD2_2': 0.00028, + } + } + }) + + monkeypatch.setattr('exotic.api.plate_solution.requests.post', fake_post) + monkeypatch.setattr('exotic.api.plate_solution.requests.get', fake_get) + monkeypatch.setattr('exotic.api.plate_solution.time.sleep', lambda _: None) + + solver = NextAstroPlateSolution(file=fits_path, directory=tmp_path, ra=210.8023, dec=54.3489, pixel_scale=1.23) + wcs_file = solver.plate_solution() + + assert wcs_file == tmp_path / 'working_artifacts' / 'wcs.fits' + header = getheader(wcs_file) + assert header['CTYPE1'] == 'RA---TAN' + assert header['CTYPE2'] == 'DEC--TAN' + + +def test_plate_solution_logs_json_via_message_logger_when_fail_warnings_suppressed(tmp_path, monkeypatch): + fits_path = _create_test_fits(tmp_path) + (tmp_path / "working_artifacts").mkdir() + logged = [] + + monkeypatch.setattr( + 'exotic.api.plate_solution.requests.post', + lambda url, data, headers, timeout: DummyResponse({'status': 'queued', 'request_id': 'abc123'}) + ) + monkeypatch.setattr( + 'exotic.api.plate_solution.requests.get', + lambda url, timeout: DummyResponse({ + 'status': 'solved', + 'solution': { + 'wcs_header': { + 'SIMPLE': True, + 'BITPIX': -64, + 'NAXIS': 2, + 'NAXIS1': 120, + 'NAXIS2': 100, + 'CTYPE1': 'RA---TAN', + 'CTYPE2': 'DEC--TAN', + } + } + }) + ) + monkeypatch.setattr('exotic.api.plate_solution.time.sleep', lambda _: None) + + solver = NextAstroPlateSolution( + file=fits_path, + directory=tmp_path, + suppress_fail_warning=True, + message_logger=logged.append + ) + + wcs_file = solver.plate_solution() + + assert wcs_file == tmp_path / 'working_artifacts' / 'wcs.fits' + assert any('NextAstro astrometry request JSON:' in message for message in logged) + assert any('NextAstro astrometry request compression:' in message for message in logged) + assert any('NextAstro astrometry submission response JSON:' in message for message in logged) + assert any('NextAstro astrometry status response JSON (solved):' in message for message in logged) + + +def test_extract_astrometry_hints_with_scale_only(tmp_path): + fits_path = _create_test_fits(tmp_path) + solver = NextAstroPlateSolution(file=fits_path, directory=tmp_path, pixel_scale=2.0) + + hints = solver._extract_astrometry_hints() + + assert hints == {'scale_arcsec_per_pix': 2.0, 'scale_tolerance_frac': 0.25} + + +def test_poll_for_solution_accepts_case_insensitive_running_status(tmp_path, monkeypatch): + fits_path = _create_test_fits(tmp_path) + solver = NextAstroPlateSolution(file=fits_path, directory=tmp_path) + + responses = iter([ + DummyResponse({'status': 'RUNNING'}), + DummyResponse({ + 'status': 'solved', + 'solution': { + 'wcs_header': { + 'SIMPLE': True, + 'BITPIX': -64, + 'NAXIS': 2, + 'NAXIS1': 120, + 'NAXIS2': 100, + 'CTYPE1': 'RA---TAN', + 'CTYPE2': 'DEC--TAN', + } + } + }) + ]) + + monkeypatch.setattr('exotic.api.plate_solution.requests.get', lambda url, timeout: next(responses)) + monkeypatch.setattr('exotic.api.plate_solution.time.sleep', lambda _: None) + + header = solver._poll_for_solution('abc123') + + assert header is not False + assert header['CTYPE1'] == 'RA---TAN' + + +def test_poll_for_solution_logs_unexpected_status(tmp_path, monkeypatch, capsys): + fits_path = _create_test_fits(tmp_path) + solver = NextAstroPlateSolution(file=fits_path, directory=tmp_path) + + monkeypatch.setattr('exotic.api.plate_solution.requests.get', + lambda url, timeout: DummyResponse({'status': 'processing'})) + + header = solver._poll_for_solution('abc123') + + assert header is False + output = capsys.readouterr().out + assert "NextAstro astrometry status response JSON (unexpected)" in output + assert '"status": "processing"' in output + + +def test_submit_solve_request_handles_non_json_response(tmp_path, monkeypatch, capsys): + fits_path = _create_test_fits(tmp_path) + solver = NextAstroPlateSolution(file=fits_path, directory=tmp_path) + + monkeypatch.setattr( + 'exotic.api.plate_solution.requests.post', + lambda url, data, headers, timeout: DummyResponse( + status_code=502, + text='bad gateway', + json_error=ValueError('not json') + ) + ) + + request_id = solver._submit_solve_request({ + 'x': [25.0], + 'y': [30.0], + 'flux': [10000.0], + 'pixel_indexing': '0-based' + }) + + assert request_id is False + output = capsys.readouterr().out + assert "NextAstro astrometry request JSON:" in output + assert "Solve response returned non-JSON response" not in output + assert "bad gateway" not in output.lower() + assert solver.last_http_status == 502 + + +def test_poll_for_solution_handles_non_json_response(tmp_path, monkeypatch, capsys): + fits_path = _create_test_fits(tmp_path) + solver = NextAstroPlateSolution(file=fits_path, directory=tmp_path) + + monkeypatch.setattr( + 'exotic.api.plate_solution.requests.get', + lambda url, timeout: DummyResponse( + status_code=200, + text='', + json_error=ValueError('not json') + ) + ) + + header = solver._poll_for_solution('abc123') + + assert header is False + output = capsys.readouterr().out + assert "Status response returned non-JSON response" in output + assert "" in output + + +def test_nova_upload_includes_astrometry_hints(tmp_path, monkeypatch): + fits_path = _create_test_fits(tmp_path) + + captured_payload = {} + + def fake_post(url, files, data, timeout): + captured_payload['url'] = url + captured_payload['request_json'] = data['request-json'] + return DummyResponse({'status': 'success', 'subid': 42}) + + monkeypatch.setattr('exotic.api.plate_solution.requests.post', fake_post) + + solver = PlateSolution(file=fits_path, directory=tmp_path, ra=150.123, dec=-2.456, + pixel_scale=1.5, radius=1.2, scale_err=30) + sub_id = solver._upload(session='session-id') + + assert sub_id == 42 + assert captured_payload['url'].endswith('/upload') + assert '"session": "session-id"' in captured_payload['request_json'] + assert '"center_ra": 150.123' in captured_payload['request_json'] + assert '"center_dec": -2.456' in captured_payload['request_json'] + assert '"radius": 1.2' in captured_payload['request_json'] + assert '"scale_units": "arcsecperpix"' in captured_payload['request_json'] + assert '"scale_type": "ev"' in captured_payload['request_json'] + assert '"scale_est": 1.5' in captured_payload['request_json'] + assert '"scale_err": 30.0' in captured_payload['request_json'] diff --git a/tests/test_nextastro_variability.py b/tests/test_nextastro_variability.py new file mode 100644 index 00000000..52eab16d --- /dev/null +++ b/tests/test_nextastro_variability.py @@ -0,0 +1,3876 @@ +import importlib.util +import sys +import types +import gzip +import json + +import numpy as np +import pytest +from tenacity import Future, RetryError + +from exotic.plate_status import PlateStatus + +fake_barycorrpy = types.ModuleType('barycorrpy') +fake_utc_tdb = types.ModuleType('barycorrpy.utc_tdb') +fake_utc_tdb.JDUTC_to_BJDTDB = lambda *args, **kwargs: None +fake_barycorrpy.utc_tdb = fake_utc_tdb + + +def _module_available(name: str) -> bool: + try: + return importlib.util.find_spec(name) is not None + except (ModuleNotFoundError, ValueError): + return False + + +def _set_stub_if_missing(name: str, module: types.ModuleType) -> None: + if not _module_available(name): + sys.modules.setdefault(name, module) + + +fake_astroalign = types.ModuleType("astroalign") +fake_astroalign.PIXEL_TOL = 1 +fake_astroquery = types.ModuleType("astroquery") +fake_astroquery_simbad = types.ModuleType("astroquery.simbad") +fake_astroquery_simbad.Simbad = type("Simbad", (), {}) +fake_astroquery_gaia = types.ModuleType("astroquery.gaia") +fake_astroquery_gaia.Gaia = type("Gaia", (), {}) +fake_imreg_dft = types.ModuleType("imreg_dft") +fake_colour_demosaicing = types.ModuleType("colour_demosaicing") +fake_colour_demosaicing.demosaicing_CFA_Bayer_bilinear = lambda *args, **kwargs: None +fake_photutils = types.ModuleType("photutils") +fake_photutils_aperture = types.ModuleType("photutils.aperture") +fake_photutils_aperture.CircularAperture = type("CircularAperture", (), {}) +fake_photutils_aperture.CircularAnnulus = type("CircularAnnulus", (), {}) +fake_photutils_detection = types.ModuleType("photutils.detection") +fake_photutils_detection.DAOStarFinder = type("DAOStarFinder", (), {}) +fake_ldtk = types.ModuleType("ldtk") +fake_ldtk.LDPSet = type("LDPSet", (), {}) +fake_ldtk.ldtk = types.SimpleNamespace(LDPSet=fake_ldtk.LDPSet) +fake_ldtk_ldmodel = types.ModuleType("ldtk.ldmodel") +fake_ldtk_ldmodel.LinearModel = type("LinearModel", (), {}) +fake_ldtk_ldmodel.QuadraticModel = type("QuadraticModel", (), {}) +fake_ldtk_ldmodel.NonlinearModel = type("NonlinearModel", (), {}) +fake_lmfit = types.ModuleType("lmfit") +fake_pylightcurve = types.ModuleType("pylightcurve") +fake_pylightcurve_models = types.ModuleType("pylightcurve.models") +fake_pylightcurve_exoplanet = types.ModuleType("pylightcurve.models.exoplanet_lc") +fake_pylightcurve_exoplanet.transit = lambda *args, **kwargs: None +fake_pyvo = types.ModuleType("pyvo") +fake_ultranest = types.ModuleType("ultranest") +fake_ultranest.ReactiveNestedSampler = type("ReactiveNestedSampler", (), {}) +fake_elca = types.ModuleType("exotic.api.elca") +fake_elca.lc_fitter = lambda *args, **kwargs: None +fake_elca.binner = lambda *args, **kwargs: None +fake_elca.transit = lambda *args, **kwargs: None +fake_elca.get_phase = lambda *args, **kwargs: None +fake_ld = types.ModuleType("exotic.api.ld") +fake_ld.LimbDarkening = type("LimbDarkening", (), {}) +fake_ld.ld_re_punct_p = lambda *args, **kwargs: None + +_set_stub_if_missing("astroalign", fake_astroalign) +_set_stub_if_missing("astroquery", fake_astroquery) +_set_stub_if_missing("astroquery.simbad", fake_astroquery_simbad) +_set_stub_if_missing("astroquery.gaia", fake_astroquery_gaia) +_set_stub_if_missing("imreg_dft", fake_imreg_dft) +_set_stub_if_missing("colour_demosaicing", fake_colour_demosaicing) +_set_stub_if_missing("photutils", fake_photutils) +_set_stub_if_missing("photutils.aperture", fake_photutils_aperture) +_set_stub_if_missing("photutils.detection", fake_photutils_detection) +_set_stub_if_missing("ldtk", fake_ldtk) +_set_stub_if_missing("ldtk.ldmodel", fake_ldtk_ldmodel) +_set_stub_if_missing("lmfit", fake_lmfit) +_set_stub_if_missing("pylightcurve", fake_pylightcurve) +_set_stub_if_missing("pylightcurve.models", fake_pylightcurve_models) +_set_stub_if_missing("pylightcurve.models.exoplanet_lc", fake_pylightcurve_exoplanet) +_set_stub_if_missing("pyvo", fake_pyvo) +_set_stub_if_missing("ultranest", fake_ultranest) +sys.modules.setdefault('barycorrpy', fake_barycorrpy) +sys.modules.setdefault('barycorrpy.utc_tdb', fake_utc_tdb) +sys.modules.setdefault("exotic.api.elca", fake_elca) +sys.modules.setdefault("exotic.api.ld", fake_ld) + +from exotic import exotic as exotic_module + + +class DummyResponse: + def __init__(self, payload, status_code=200): + self._payload = payload + self.status_code = status_code + + def json(self): + return self._payload + + def raise_for_status(self): + if self.status_code >= 400: + raise RuntimeError(f"HTTP {self.status_code}") + + +def _decode_request_body(body, headers): + encoding = headers["Content-Encoding"] + if encoding == "gzip": + return json.loads(gzip.decompress(body).decode("utf-8")) + if encoding == "zstd": + zstandard = pytest.importorskip("zstandard") + return json.loads(zstandard.ZstdDecompressor().decompress(body).decode("utf-8")) + raise AssertionError(f"Unexpected content encoding: {encoding}") + + +def test_nextastro_variability_logs_json_request_and_response(monkeypatch): + captured = {} + logged = [] + + def fake_post(url, data, headers, timeout): + captured['url'] = url + captured['json'] = _decode_request_body(data, headers) + captured['headers'] = headers + captured['timeout'] = timeout + return DummyResponse([ + {'is_in_vsx': 0}, + {'is_in_vsx': 1}, + ]) + + monkeypatch.setattr(exotic_module.requests, 'post', fake_post) + monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) + + variability_flags = exotic_module.nextastro_variability_test([(10.1, -11.2), (22.3, -33.4)]) + + assert captured['url'].endswith('/variability_test') + assert captured['timeout'] == 30 + assert captured['headers']['Content-Type'] == 'application/json' + assert captured['headers']['Content-Encoding'] in {'gzip', 'zstd'} + assert captured['json'] == [{'ra': 10.1, 'dec': -11.2}, {'ra': 22.3, 'dec': -33.4}] + assert variability_flags == [False, True] + assert any('NextAstro variability request JSON:' in message for message in logged) + assert any('NextAstro variability request compression:' in message for message in logged) + assert any('NextAstro variability response JSON:' in message for message in logged) + + +def test_nextastro_variability_retries_zstd_415_once_with_gzip(monkeypatch): + logged = [] + encodings = [] + + def fake_build_compressed_json_request(payload, content_encoding=None): + encoding = content_encoding or 'zstd' + body = json.dumps(payload).encode('utf-8') + headers = { + 'Content-Type': 'application/json', + 'Content-Encoding': encoding, + } + return body, headers, encoding, len(body), len(body) + + def fake_post(url, data, headers, timeout): + encodings.append(headers['Content-Encoding']) + if headers['Content-Encoding'] == 'zstd': + return DummyResponse(None, status_code=415) + return DummyResponse([{'is_in_vsx': 1}]) + + monkeypatch.setattr(exotic_module, 'build_compressed_json_request', fake_build_compressed_json_request) + monkeypatch.setattr(exotic_module.requests, 'post', fake_post) + monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) + + variability_flags = exotic_module.nextastro_variability_test([(10.1, -11.2)]) + + assert variability_flags == [True] + assert encodings == ['zstd', 'gzip'] + assert any('rejected zstd-compressed request (HTTP 415)' in message for message in logged) + + +def test_nextastro_variability_caps_retry_attempts_at_five(monkeypatch): + attempts = [] + + def fake_build_compressed_json_request(payload, content_encoding=None): + encoding = content_encoding or 'gzip' + body = b'{}' + headers = { + 'Content-Type': 'application/json', + 'Content-Encoding': encoding, + } + return body, headers, encoding, len(body), len(body) + + def fake_post(url, data, headers, timeout): + attempts.append(headers['Content-Encoding']) + return DummyResponse(None, status_code=502) + + monkeypatch.setattr(exotic_module, 'build_compressed_json_request', fake_build_compressed_json_request) + monkeypatch.setattr(exotic_module.requests, 'post', fake_post) + monkeypatch.setattr(exotic_module.nextastro_variability_test.retry, 'sleep', lambda _: None) + + with pytest.raises(RetryError) as excinfo: + exotic_module.nextastro_variability_test([(10.1, -11.2)]) + + assert len(attempts) == 5 + assert excinfo.value.last_attempt.attempt_number == 5 + + +def test_nextastro_vsx_query_boxes_split_ra_wrap(): + boxes = exotic_module.nextastro_vsx_query_boxes(359.9, 0.0, 0.2) + + np.testing.assert_allclose(boxes, [ + (359.7, 360.0, -0.2, 0.2), + (0.0, 0.1, -0.2, 0.2), + ]) + + +def test_nextastro_vsx_field_query_normalizes_rows(monkeypatch): + captured = {} + + def fake_post(url, json, timeout): + captured['url'] = url + captured['json'] = json + captured['timeout'] = timeout + return DummyResponse({ + 'columns': ['oid', 'name', 'ra_deg', 'dec_deg', 'var_type', 'mag1', 'mag1_band'], + 'count': 1, + 'row_format': 'objects', + 'rows': [{ + 'oid': 123, + 'name': 'Cached Variable', + 'ra_deg': 10.1, + 'dec_deg': -20.2, + 'var_type': 'EA', + 'mag1': 12.3, + 'mag1_band': 'V', + }], + }) + + monkeypatch.setattr(exotic_module.requests, 'post', fake_post) + + rows = exotic_module.nextastro_vsx_field_query(10.0, -20.0, 0.25) + + assert captured['url'].endswith('/vsx_query') + assert captured['timeout'] == 30 + assert captured['json']['compact'] is False + assert rows[0]['Name'] == 'Cached Variable' + assert rows[0]['OID'] == 123 + assert rows[0]['RA2000'] == pytest.approx(10.1) + assert rows[0]['Declination2000'] == pytest.approx(-20.2) + assert rows[0]['VariabilityType'] == 'EA' + assert rows[0]['MaxMag'] == '12.3 V' + assert rows[0]['_vsx_source'] == 'nextastro_cache' + assert rows[0]['_vsx_has_full_metadata'] is False + + +def test_vsx_field_query_cache_first_falls_back_when_cache_empty(monkeypatch): + calls = [] + monkeypatch.setattr(exotic_module, 'nextastro_vsx_field_query', lambda *args: []) + monkeypatch.setattr( + exotic_module, + 'vsx_field_query', + lambda *args, **kwargs: calls.append((args, kwargs)) or [{'Name': 'AAVSO Variable'}], + ) + + rows = exotic_module.vsx_field_query_with_preference( + 10.0, + -20.0, + 0.25, + use_nextastro_vsx_cache_first=True, + ) + + assert rows == [{'Name': 'AAVSO Variable'}] + assert len(calls) == 1 + + +def test_vsx_field_query_cache_first_enriches_period_and_amplitude(monkeypatch): + monkeypatch.setattr( + exotic_module, + 'nextastro_vsx_field_query', + lambda *args: [{ + 'OID': 123, + 'Name': 'Cached Name', + 'RA2000': 10.1, + 'Declination2000': -20.2, + 'VariabilityType': 'EA', + '_vsx_source': 'nextastro_cache', + }], + ) + monkeypatch.setattr( + exotic_module, + 'vsx_field_query', + lambda *args, **kwargs: [{ + 'OID': '123', + 'Name': 'AAVSO Name', + 'RA2000': '10.1000', + 'Declination2000': '-20.2000', + 'Period': '2.5', + 'MaxMag': '12.0 V', + 'MinMag': '12.4 V', + }], + ) + + rows = exotic_module.vsx_field_query_with_preference( + 10.0, + -20.0, + 0.25, + use_nextastro_vsx_cache_first=True, + ) + + assert rows[0]['Name'] == 'AAVSO Name' + assert rows[0]['Period'] == '2.5' + assert exotic_module.vsx_object_amplitude_mag(rows[0]) == pytest.approx(0.4) + assert rows[0]['_vsx_source'] == 'nextastro_cache+aavso_metadata' + + +def test_vsx_field_query_cache_first_uses_full_nextastro_metadata_without_aavso(monkeypatch): + cached_row = exotic_module.normalize_nextastro_vsx_row({ + 'oid': 123, + 'name': 'Cached Full Variable', + 'ra_deg': 10.1, + 'dec_deg': -20.2, + 'var_type': 'EA', + 'period_days': 2.5, + 'amplitude_mag': 0.4, + 'max_mag': 12.0, + 'max_passband': 'V', + 'min_mag': 12.4, + 'min_passband': 'V', + }) + monkeypatch.setattr( + exotic_module, + 'nextastro_vsx_field_query', + lambda *args: [cached_row], + ) + + def unexpected_aavso_call(*args, **kwargs): + raise AssertionError('AAVSO should not be called for the full NextAstro schema') + + monkeypatch.setattr(exotic_module, 'vsx_field_query', unexpected_aavso_call) + + rows = exotic_module.vsx_field_query_with_preference( + 10.0, + -20.0, + 0.25, + use_nextastro_vsx_cache_first=True, + ) + + assert rows[0]['_vsx_has_full_metadata'] is True + assert rows[0]['Period'] == 2.5 + assert rows[0]['Amplitude'] == 0.4 + assert rows[0]['MaxMag'] == '12.0 V' + assert rows[0]['MinMag'] == '12.4 V' + + +def test_nextastro_photometry_catalog_match_prefers_requested_filter(): + catalog = { + 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag', 'g', 'dg'], + 'count': 2, + 'row_format': 'objects', + 'rows': [ + { + 'id': 1, + 'source_id': 111, + 'ra': 10.0001, + 'dec': 20.0001, + 'Vmag': None, + 'err_Vmag': None, + 'g': 12.1, + 'dg': 0.02, + }, + { + 'id': 2, + 'source_id': 222, + 'ra': 10.0002, + 'dec': 20.0002, + 'Vmag': 12.3, + 'err_Vmag': 0.04, + 'g': 12.0, + 'dg': 0.02, + }, + ], + } + + match = exotic_module.nextastro_photometry_catalog_match(catalog, 10.0, 20.0, 'CV') + + assert match['source_id'] == 222 + assert match['mag'] == pytest.approx(12.3) + assert match['error'] == pytest.approx(0.04) + assert match['mag_band'] == 'V' + assert match['separation_arcsec'] > 0 + + +def test_nextastro_photometry_catalog_match_floors_zero_magnitude_error(): + catalog = { + 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag'], + 'count': 1, + 'row_format': 'objects', + 'rows': [ + { + 'id': 1, + 'source_id': 111, + 'ra': 10.0001, + 'dec': 20.0001, + 'Vmag': 12.3, + 'err_Vmag': 0.0, + }, + ], + } + + match = exotic_module.nextastro_photometry_catalog_match(catalog, 10.0, 20.0, 'CV') + + assert match['error'] == pytest.approx(0.001) + + +def test_nextastro_photometry_catalog_match_accepts_relaxed_v_magnitude_error(): + catalog = { + 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag'], + 'count': 1, + 'row_format': 'objects', + 'rows': [ + { + 'id': 1, + 'source_id': 111, + 'ra': 10.0001, + 'dec': 20.0001, + 'Vmag': 12.3, + 'err_Vmag': 0.051, + }, + ], + } + + match = exotic_module.nextastro_photometry_catalog_match(catalog, 10.0, 20.0, 'CV') + + assert match['mag_band'] == 'V' + assert match['error'] == pytest.approx(0.051) + assert match['uses_relaxed_bv_error_limit'] is True + + +def test_nextastro_photometry_catalog_match_ignores_over_30_magnitudes(): + catalog = { + 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag'], + 'count': 1, + 'row_format': 'objects', + 'rows': [ + { + 'id': 1, + 'source_id': 111, + 'ra': 10.0001, + 'dec': 20.0001, + 'Vmag': 99.99, + 'err_Vmag': 99.99, + }, + ], + } + + match = exotic_module.nextastro_photometry_catalog_match(catalog, 10.0, 20.0, 'CV') + + assert match is None + + +def test_nextastro_photometry_catalog_match_honors_scale_aware_radius(): + catalog = { + 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag'], + 'count': 1, + 'row_format': 'objects', + 'rows': [ + { + 'id': 1, + 'source_id': 111, + 'ra': 10.001, + 'dec': 20.0, + 'Vmag': 12.3, + 'err_Vmag': 0.02, + }, + ], + } + + default_match = exotic_module.nextastro_photometry_catalog_match( + catalog, + 10.0, + 20.0, + 'CV', + ) + scale_aware_match = exotic_module.nextastro_photometry_catalog_match( + catalog, + 10.0, + 20.0, + 'CV', + max_separation_arcsec=5.2, + ) + + assert default_match is None + assert scale_aware_match['source_id'] == 111 + assert 2.0 < scale_aware_match['separation_arcsec'] < 5.2 + + +def test_nextastro_catalog_match_radius_uses_one_pixel_with_two_arcsec_floor(): + assert exotic_module.nextastro_catalog_match_radius_arcsec(None) == pytest.approx(2.0) + assert exotic_module.nextastro_catalog_match_radius_arcsec(1.4) == pytest.approx(2.0) + assert exotic_module.nextastro_catalog_match_radius_arcsec(5.153485) == pytest.approx(5.153485) + + +def test_direct_selected_catalog_candidate_uses_targeted_scale_aware_lookup(monkeypatch): + calls = [] + + def fake_lookup(ra, dec, observed_filter, radius_arcsec=2.0): + calls.append((ra, dec, observed_filter, radius_arcsec)) + return { + 'id': 185212647, + 'source_id': None, + 'mag': 10.038, + 'error': 0.027, + 'mag_band': 'V', + 'catalog_ra': 294.68751, + 'catalog_dec': 31.360037, + 'separation_arcsec': 2.643594, + } + + monkeypatch.setattr(exotic_module, 'nextastro_photometry_for_coordinate', fake_lookup) + + candidate = exotic_module.build_direct_selected_catalog_candidate( + [[232.0, 348.0]], + [(294.68834158880503, 31.36022406806484)], + {'row_format': 'objects', 'rows': []}, + 0, + observed_filter='CV', + match_radius_arcsec=5.153485, + ) + + assert calls == [(294.68834158880503, 31.36022406806484, 'CV', 5.153485)] + assert candidate['source'] == 'direct_catalog' + assert candidate['star']['mag'] == pytest.approx(10.038) + assert candidate['star']['mag_band'] == 'V' + assert candidate['star']['pos'] == [232.0, 348.0] + + +def test_reported_stellar_variability_band_distinguishes_clearv_from_catalog_v(): + for observed_filter in ( + 'MObs CV', + 'CV', + 'Clear', + 'Luminance', + 'Photographic G', + 'Gaia G', + 'G', + 'G1', + 'G2', + ): + assert exotic_module.reported_stellar_variability_band( + observed_filter, 'V' + ) == 'ClearV' + assert exotic_module.reported_stellar_variability_band('bv', 'V') == 'V' + for observed_filter in ('R', 'SR', 'rp'): + assert exotic_module.reported_stellar_variability_band( + observed_filter, 'r' + ) == 'rp' + + +def test_selected_comparison_direct_catalog_match_precedes_derived_fallback(): + selected = exotic_module.choose_selected_comp_catalog_reference_candidate( + [ + { + 'source': 'field_derived', + 'error': 0.010, + 'star': {'mag_band': 'V', 'error': 0.010}, + }, + { + 'source': 'direct_catalog', + 'error': 0.027, + 'star': {'mag_band': 'V', 'error': 0.027}, + }, + ], + observed_filter='CV', + ) + + assert selected['source'] == 'direct_catalog' + + +def test_aavso_vsp_band_for_filter_uses_observed_filter_aliases(): + assert exotic_module.aavso_vsp_band_for_filter('MObs CV') == 'V' + assert exotic_module.aavso_vsp_band_for_filter('Clear (unfiltered) reduced to V sequence') == 'V' + assert exotic_module.aavso_vsp_band_for_filter('Photographic G') == 'V' + assert exotic_module.aavso_vsp_band_for_filter('Gaia G') == 'V' + assert exotic_module.aavso_vsp_band_for_filter('Cousins R') == 'Rc' + assert exotic_module.aavso_vsp_band_for_filter('Sloan g') == 'SG' + + +def test_nextastro_catalog_band_tokens_keep_bessell_sloan_and_clearv_distinct(): + assert exotic_module.nextastro_photometry_band_candidates('bv') == [ + ('Vmag', 'err_Vmag', 'V') + ] + assert exotic_module.nextastro_photometry_band_candidates('bb') == [ + ('Bmag', 'err_Bmag', 'B') + ] + assert exotic_module.nextastro_photometry_band_candidates('Sloan g') == [('g', 'dg', 'g')] + for observed_filter in ('G', 'Photographic G', 'Gaia G', 'G1', 'G2'): + assert exotic_module.nextastro_photometry_band_candidates(observed_filter) == [ + ('Vmag', 'err_Vmag', 'V') + ] + assert exotic_module.catalog_band_priority('V', 'bv') == 0 + assert exotic_module.catalog_band_priority('B', 'bb') == 0 + assert exotic_module.catalog_band_priority('g', 'Sloan g') == 0 + assert exotic_module.catalog_band_priority('g', 'G') == 1 + assert exotic_module.catalog_band_priority('g', 'Gaia G') == 1 + assert exotic_module.catalog_band_priority('V', 'G') == 0 + assert exotic_module.catalog_band_priority('V', 'Gaia G') == 0 + + +@pytest.mark.parametrize( + ('observed_filter', 'expected_labels'), + [ + ('u', ['u-g', 'B-V', 'BP-RP']), + ('Johnson U', ['u-g', 'B-V', 'BP-RP']), + ('B', ['B-V', 'BP-RP']), + ('Photographic B', ['B-V', 'BP-RP']), + ('V', ['B-V', 'BP-RP']), + ('Sloan g', ['g-r', 'B-V', 'BP-RP']), + ('Sloan r', ['r-i', 'B-V', 'BP-RP']), + ('Cousins R', ['r-i', 'B-V', 'BP-RP']), + ('Sloan i', ['r-i', 'B-V', 'BP-RP']), + ('Cousins I', ['r-i', 'B-V', 'BP-RP']), + ('Sloan z', ['i-z', 'B-V', 'BP-RP']), + ('CV', ['B-V', 'BP-RP']), + ('Clear', ['B-V', 'BP-RP']), + ('Luminance', ['B-V', 'BP-RP']), + ('Photographic G', ['B-V', 'BP-RP']), + ('Gaia G', ['B-V', 'BP-RP']), + ('G', ['B-V', 'BP-RP']), + ('G1', ['B-V', 'BP-RP']), + ('G2', ['B-V', 'BP-RP']), + ('Unknown', ['B-V', 'BP-RP']), + ], +) +def test_all_filters_use_filter_specific_color_then_universal_fallbacks( + observed_filter, expected_labels): + pairs = exotic_module.nextastro_color_candidate_pairs(observed_filter) + assert [pair[2] for pair in pairs] == expected_labels + + +@pytest.mark.parametrize( + ('observed_filter', 'primary_label', 'primary_columns'), + [ + ('u', 'u-g', ('umag',)), + ('B', 'B-V', ()), + ('V', 'B-V', ()), + ('Sloan g', 'g-r', ('g',)), + ('Sloan r', 'r-i', ('r',)), + ('Sloan i', 'r-i', ('r',)), + ('Sloan z', 'i-z', ('i',)), + ('Clear', 'B-V', ()), + ('Photographic G', 'B-V', ()), + ('Gaia G', 'B-V', ()), + ], +) +def test_all_filters_fall_back_from_specific_color_to_bv_then_bp_rp( + observed_filter, primary_label, primary_columns): + row = { + 'Bmag': 13.0, + 'Vmag': 12.5, + 'umag': 13.8, + 'g': 12.8, + 'r': 12.2, + 'i': 12.0, + 'z': 11.8, + 'bp_rp': 1.1, + } + assert exotic_module.nextastro_catalog_color(row, observed_filter)['label'] == primary_label + + for column in primary_columns: + row[column] = None + assert exotic_module.nextastro_catalog_color(row, observed_filter)['label'] == 'B-V' + + row['Bmag'] = None + assert exotic_module.nextastro_catalog_color(row, observed_filter) == { + 'color': pytest.approx(1.1), + 'label': 'BP-RP', + 'first_column': 'bp_rp', + 'second_column': None, + } + + +def test_clearv_catalog_color_derives_bp_rp_from_gaia_magnitudes(): + color = exotic_module.nextastro_catalog_color( + { + 'Vmag': 12.5, + 'phot_bp_mean_mag': 13.4, + 'phot_rp_mean_mag': 12.1, + }, + 'Gaia G', + ) + + assert color == { + 'color': pytest.approx(1.3), + 'label': 'BP-RP', + 'first_column': 'phot_bp_mean_mag', + 'second_column': 'phot_rp_mean_mag', + } + + +def test_nextastro_gaia_bp_rp_lookup_is_cached(monkeypatch): + captured = [] + + def fake_get(url, params, timeout): + captured.append((url, params, timeout)) + return DummyResponse({ + 'gaia': { + 'source_id': 123456, + 'separation_arcsec': 0.2, + 'phot_bp_mean_mag': 13.4, + 'phot_rp_mean_mag': 12.1, + 'bp_rp': 1.3, + }, + }) + + exotic_module._cached_nextastro_gaia_bp_rp.cache_clear() + monkeypatch.setattr(exotic_module.requests, 'get', fake_get) + + first = exotic_module.nextastro_gaia_bp_rp_for_coordinate(10.12345678, -20.25) + second = exotic_module.nextastro_gaia_bp_rp_for_coordinate(10.12345678, -20.25) + + assert first == second + assert first['color'] == pytest.approx(1.3) + assert first['label'] == 'BP-RP' + assert first['catalog_source'] == 'NextAstro Gaia DR3' + assert captured == [( + exotic_module.NEXTASTRO_GAIA_DISTPM_ENDPOINT, + {'ra': 10.1234568, 'dec': -20.25}, + exotic_module.NEXTASTRO_GAIA_COLOR_LOOKUP_TIMEOUT_SECONDS, + )] + + +def test_local_catalog_color_does_not_query_gaia(monkeypatch): + monkeypatch.setattr( + exotic_module, + 'nextastro_gaia_bp_rp_for_coordinate', + lambda *args, **kwargs: pytest.fail('Gaia should not be queried when B-V is available.'), + ) + state = {'remaining': 3, 'attempted': 0, 'matched': 0} + + color = exotic_module.nextastro_catalog_color_with_gaia_fallback( + {'ra': 10.0, 'dec': 20.0, 'Bmag': 13.0, 'Vmag': 12.5}, + 'V', + lookup_state=state, + ) + + assert color['label'] == 'B-V' + assert color['color'] == pytest.approx(0.5) + assert state == {'remaining': 3, 'attempted': 0, 'matched': 0} + + +def test_nearest_catalog_color_row_uses_gaia_bp_rp_last_resort(monkeypatch): + calls = [] + + def fake_gaia_lookup(ra, dec, max_separation_arcsec): + calls.append((ra, dec, max_separation_arcsec)) + return {'color': 1.25, 'label': 'BP-RP'} + + monkeypatch.setattr(exotic_module, 'nextastro_gaia_bp_rp_for_coordinate', fake_gaia_lookup) + state = {'remaining': 3, 'attempted': 0, 'matched': 0} + catalog = { + 'rows': [ + {'source_id': 42, 'ra': 10.00001, 'dec': 20.0, 'Vmag': 12.5}, + ], + } + + match = exotic_module.nextastro_catalog_nearest_color_row( + catalog, + 10.0, + 20.0, + 'V', + gaia_lookup_state=state, + gaia_match_radius_arcsec=1.5, + ) + + assert match['source_id'] == 42 + assert match['color'] == {'color': 1.25, 'label': 'BP-RP'} + assert calls == [(10.00001, 20.0, 1.5)] + assert state == {'remaining': 2, 'attempted': 1, 'matched': 1} + + +@pytest.mark.parametrize( + ('observed_filter', 'magnitude_column', 'error_column'), + [ + ('u', 'umag', 'err_umag'), + ('B', 'Bmag', 'err_Bmag'), + ('V', 'Vmag', 'err_Vmag'), + ('Sloan g', 'g', 'dg'), + ('Sloan r', 'r', 'dr'), + ('Sloan i', 'i', 'di'), + ('Sloan z', 'z', 'dz'), + ], +) +def test_nextastro_catalog_match_never_falls_back_to_another_band( + observed_filter, magnitude_column, error_column): + row = { + 'id': 1, + 'ra': 10.0, + 'dec': 20.0, + 'Bmag': 12.1, + 'err_Bmag': 0.01, + 'Vmag': 12.2, + 'err_Vmag': 0.01, + 'umag': 12.3, + 'err_umag': 0.01, + 'g': 12.4, + 'dg': 0.01, + 'r': 12.5, + 'dr': 0.01, + 'i': 12.6, + 'di': 0.01, + 'z': 12.7, + 'dz': 0.01, + } + row[magnitude_column] = None + row[error_column] = None + + match = exotic_module.nextastro_photometry_catalog_match( + {'row_format': 'objects', 'rows': [row]}, + 10.0, + 20.0, + observed_filter, + ) + + assert match is None + + +def test_nextastro_catalog_match_uses_relaxed_error_only_as_bv_fallback(): + catalog = { + 'row_format': 'objects', + 'rows': [ + { + 'id': 1, 'ra': 10.00001, 'dec': 20.0, + 'Vmag': 12.1, 'err_Vmag': 0.07, + }, + { + 'id': 2, 'ra': 10.00010, 'dec': 20.0, + 'Vmag': 12.2, 'err_Vmag': 0.03, + }, + ], + } + + preferred = exotic_module.nextastro_photometry_catalog_match(catalog, 10.0, 20.0, 'V') + assert preferred['id'] == 2 + assert preferred['uses_relaxed_bv_error_limit'] is False + + relaxed_v = exotic_module.nextastro_photometry_catalog_match( + {'row_format': 'objects', 'rows': [catalog['rows'][0]]}, + 10.0, + 20.0, + 'bv', + ) + assert relaxed_v['id'] == 1 + assert relaxed_v['error'] == pytest.approx(0.07) + assert relaxed_v['uses_relaxed_bv_error_limit'] is True + + relaxed_b = exotic_module.nextastro_photometry_catalog_match( + { + 'row_format': 'objects', + 'rows': [{'id': 3, 'ra': 10.0, 'dec': 20.0, 'Bmag': 13.0, 'err_Bmag': 0.10}], + }, + 10.0, + 20.0, + 'bb', + ) + assert relaxed_b['id'] == 3 + assert relaxed_b['uses_relaxed_bv_error_limit'] is True + + rejected_g = exotic_module.nextastro_photometry_catalog_match( + { + 'row_format': 'objects', + 'rows': [{'id': 4, 'ra': 10.0, 'dec': 20.0, 'g': 13.0, 'dg': 0.051}], + }, + 10.0, + 20.0, + 'Sloan g', + ) + assert rejected_g is None + + rejected_v = exotic_module.nextastro_photometry_catalog_match( + { + 'row_format': 'objects', + 'rows': [{'id': 5, 'ra': 10.0, 'dec': 20.0, 'Vmag': 13.0, 'err_Vmag': 0.101}], + }, + 10.0, + 20.0, + 'V', + ) + assert rejected_v is None + + +def test_nextastro_photometry_for_coordinate_uses_single_object_endpoint(monkeypatch): + captured = {} + + def fake_post(url, json, timeout): + captured['url'] = url + captured['json'] = json + captured['timeout'] = timeout + return DummyResponse({ + 'columns': list(exotic_module.NEXTASTRO_PHOTOMETRY_COLUMNS), + 'match': { + 'id': 9, + 'source_id': 12345, + 'ra': 10.00001, + 'dec': -20.00001, + 'Vmag': 11.2, + 'err_Vmag': 0.03, + }, + 'separation_arcsec': 0.05, + }) + + monkeypatch.setattr(exotic_module.requests, 'post', fake_post) + monkeypatch.setattr(exotic_module, 'log_info', lambda *args, **kwargs: None) + + match = exotic_module.nextastro_photometry_for_coordinate(10.0, -20.0, 'V') + + assert captured['url'] == 'https://photometry.nextastro.org/single_object' + assert captured['json']['ra'] == pytest.approx(10.0) + assert captured['json']['dec'] == pytest.approx(-20.0) + assert captured['json']['radius_arcsec'] == pytest.approx(2.0) + assert captured['json']['columns'] == [ + 'id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag', + ] + assert captured['json']['required_columns'] == ['Vmag', 'err_Vmag'] + assert captured['timeout'] == 30 + assert match['source_id'] == 12345 + assert match['mag'] == pytest.approx(11.2) + assert match['mag_band'] == 'V' + + +def test_nextastro_photometry_for_coordinates_uses_objects_query_endpoint(monkeypatch): + captured = {} + + def fake_post(url, json, timeout): + captured['url'] = url + captured['json'] = json + captured['timeout'] = timeout + return DummyResponse({ + 'columns': list(exotic_module.NEXTASTRO_PHOTOMETRY_COLUMNS), + 'count': 1, + 'results': [ + { + 'key': '0', + 'ra': 10.0, + 'dec': -20.0, + 'match': { + 'id': 9, + 'source_id': 12345, + 'ra': 10.00001, + 'dec': -20.00001, + 'Vmag': 11.2, + 'err_Vmag': 0.03, + }, + 'separation_arcsec': 0.05, + }, + { + 'key': '1', + 'ra': 11.0, + 'dec': -21.0, + 'match': None, + 'separation_arcsec': None, + }, + ], + }) + + monkeypatch.setattr(exotic_module.requests, 'post', fake_post) + monkeypatch.setattr(exotic_module, 'log_info', lambda *args, **kwargs: None) + + matches = exotic_module.nextastro_photometry_for_coordinates( + [(10.0, -20.0), (11.0, -21.0)], + 'V', + ) + + assert captured['url'] == 'https://photometry.nextastro.org/objects_query' + assert captured['json']['objects'] == [ + {'key': '0', 'ra': 10.0, 'dec': -20.0}, + {'key': '1', 'ra': 11.0, 'dec': -21.0}, + ] + assert captured['json']['radius_arcsec'] == pytest.approx(2.0) + assert captured['json']['columns'] == [ + 'id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag', + ] + assert captured['json']['required_columns'] == ['Vmag', 'err_Vmag'] + assert captured['timeout'] == 30 + assert matches[0]['source_id'] == 12345 + assert matches[0]['mag_band'] == 'V' + assert matches[1] is None + + +def test_nextastro_photometry_for_coordinates_routes_single_target_to_single_object(monkeypatch): + calls = [] + expected_match = {'source_id': 12345, 'mag': 11.2, 'error': 0.03, 'mag_band': 'V'} + + def fake_single(ra, dec, obs_filter, radius_arcsec=2.0): + calls.append((ra, dec, obs_filter, radius_arcsec)) + return expected_match + + monkeypatch.setattr(exotic_module, 'nextastro_photometry_for_coordinate', fake_single) + monkeypatch.setattr( + exotic_module, + 'nextastro_photometry_objects_query', + lambda *args, **kwargs: pytest.fail('one target should not use /objects_query'), + ) + + matches = exotic_module.nextastro_photometry_for_coordinates([(10.0, -20.0)], 'V') + + assert calls == [(10.0, -20.0, 'V', 2.0)] + assert matches == [expected_match] + + +def test_merge_nextastro_calibration_stars_batches_missing_field_matches(monkeypatch): + calls = [] + + def fake_matches(coordinates, obs_filter, radius_arcsec=2.0): + calls.append((coordinates, obs_filter, radius_arcsec)) + return [ + { + 'source_id': 101, + 'id': 1, + 'mag': 11.2, + 'error': 0.03, + 'mag_band': 'V', + 'catalog_ra': 10.0, + 'catalog_dec': -20.0, + 'separation_arcsec': 0.1, + 'catalog_row': {'source_id': 101, 'ra': 10.0, 'dec': -20.0}, + }, + { + 'source_id': 202, + 'id': 2, + 'mag': 12.1, + 'error': 0.04, + 'mag_band': 'V', + 'catalog_ra': 11.0, + 'catalog_dec': -21.0, + 'separation_arcsec': 0.2, + 'catalog_row': {'source_id': 202, 'ra': 11.0, 'dec': -21.0}, + }, + ] + + monkeypatch.setattr( + exotic_module, + 'nextastro_photometry_for_coordinates', + fake_matches, + ) + monkeypatch.setattr(exotic_module, 'log_info', lambda *args, **kwargs: None) + + calibration_stars = exotic_module.merge_nextastro_calibration_stars( + comp_stars=[[100, 200], [130, 230]], + comp_ra_dec=[(10.0, -20.0), (11.0, -21.0)], + obs_filter='V', + existing_comp_stars={}, + field_catalog=None, + ) + + assert calls == [([(10.0, -20.0), (11.0, -21.0)], 'V', 2.0)] + assert list(calibration_stars) == ['NextAstro-101', 'NextAstro-202'] + + +def test_merge_nextastro_calibration_stars_adds_non_vsp_metadata(): + catalog = { + 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag'], + 'count': 1, + 'row_format': 'objects', + 'rows': [ + { + 'id': 9, + 'source_id': 12345, + 'ra': 10.00001, + 'dec': -20.00001, + 'Vmag': 11.2, + 'err_Vmag': 0.03, + } + ], + } + + calibration_stars = exotic_module.merge_nextastro_calibration_stars( + comp_stars=[[100, 200]], + comp_ra_dec=[(10.0, -20.0)], + obs_filter='V', + existing_comp_stars={}, + field_catalog=catalog, + ) + + assert list(calibration_stars) == ['NextAstro-12345'] + calibration = calibration_stars['NextAstro-12345'] + assert calibration['is_aavso_vsp'] is False + assert calibration['catalog_source'] == 'NextAstro photometry catalog' + assert calibration['ra'] == pytest.approx(10.0) + assert calibration['dec'] == pytest.approx(-20.0) + assert calibration['mag'] == pytest.approx(11.2) + assert calibration['error'] == pytest.approx(0.03) + assert calibration['observed_filter'] == 'V' + + +def test_merge_nextastro_calibration_stars_uses_image_scale_match_radius(): + catalog = { + 'row_format': 'objects', + 'rows': [{ + 'id': 9, + 'source_id': 12345, + 'ra': 10.001, + 'dec': 20.0, + 'Vmag': 10.038, + 'err_Vmag': 0.027, + }], + } + + calibration_stars = exotic_module.merge_nextastro_calibration_stars( + comp_stars=[[232, 348]], + comp_ra_dec=[(10.0, 20.0)], + obs_filter='MObs CV', + field_catalog=catalog, + match_radius_arcsec=5.153485, + ) + + assert list(calibration_stars) == ['NextAstro-12345'] + assert calibration_stars['NextAstro-12345']['mag'] == pytest.approx(10.038) + assert calibration_stars['NextAstro-12345']['mag_band'] == 'V' + assert calibration_stars['NextAstro-12345']['separation_arcsec'] > 2.0 + + +def test_merge_nextastro_calibration_stars_deduplicates_catalog_source_ids(): + catalog = { + 'columns': ['id', 'source_id', 'ra', 'dec', 'Vmag', 'err_Vmag'], + 'count': 1, + 'row_format': 'objects', + 'rows': [{ + 'id': 9, + 'source_id': 12345, + 'ra': 10.0, + 'dec': -20.0, + 'Vmag': 11.2, + 'err_Vmag': 0.03, + }], + } + + calibration_stars = exotic_module.merge_nextastro_calibration_stars( + comp_stars=[[100, 200], [130, 230]], + comp_ra_dec=[(10.0, -20.0), (10.0001, -20.0001)], + obs_filter='V', + existing_comp_stars={}, + field_catalog=catalog, + ) + + assert list(calibration_stars) == ['NextAstro-12345'] + + +def test_fetch_aavso_vsp_chart_retries_malformed_json_five_times_then_succeeds(monkeypatch): + payload = {'chartid': 'X-RETRY', 'photometry': []} + responses = [None] * exotic_module.AAVSO_VSP_MAX_RETRIES + [DummyResponse(payload)] + request_timeouts = [] + sleep_delays = [] + log_messages = [] + + class InvalidJSONResponse: + def raise_for_status(self): + return None + + def json(self): + return json.loads('') + + def fake_get(url, timeout): + request_timeouts.append(timeout) + response = responses.pop(0) + return InvalidJSONResponse() if response is None else response + + monkeypatch.setattr(exotic_module.requests, 'get', fake_get) + monkeypatch.setattr(exotic_module, 'sleep', sleep_delays.append) + monkeypatch.setattr( + exotic_module, + 'log_info', + lambda message, **kwargs: log_messages.append((message, kwargs)), + ) + + assert exotic_module.fetch_aavso_vsp_chart('https://example.invalid/vsp') == payload + assert request_timeouts == [ + exotic_module.AAVSO_VSP_REQUEST_TIMEOUT_SECONDS + ] * (exotic_module.AAVSO_VSP_MAX_RETRIES + 1) + assert sleep_delays == [ + exotic_module.AAVSO_VSP_RETRY_DELAY_SECONDS + ] * exotic_module.AAVSO_VSP_MAX_RETRIES + assert 'attempt 1/6' in log_messages[0][0] + assert 'attempt 5/6' in log_messages[-1][0] + assert all(kwargs.get('warn') is True for _, kwargs in log_messages) + + +def test_fetch_aavso_vsp_chart_raises_after_five_failed_retries(monkeypatch): + request_count = 0 + sleep_delays = [] + + class InvalidJSONResponse: + def raise_for_status(self): + return None + + def json(self): + return json.loads('') + + def fake_get(url, timeout): + nonlocal request_count + request_count += 1 + assert timeout == exotic_module.AAVSO_VSP_REQUEST_TIMEOUT_SECONDS + return InvalidJSONResponse() + + monkeypatch.setattr(exotic_module.requests, 'get', fake_get) + monkeypatch.setattr(exotic_module, 'sleep', sleep_delays.append) + monkeypatch.setattr(exotic_module, 'log_info', lambda *args, **kwargs: None) + + with pytest.raises(exotic_module.AAVSOVSPUnavailableError) as exc_info: + exotic_module.fetch_aavso_vsp_chart('https://example.invalid/vsp') + + assert request_count == exotic_module.AAVSO_VSP_MAX_RETRIES + 1 + assert sleep_delays == [ + exotic_module.AAVSO_VSP_RETRY_DELAY_SECONDS + ] * exotic_module.AAVSO_VSP_MAX_RETRIES + assert 'after 6 attempts (5 retries)' in str(exc_info.value) + assert 'JSONDecodeError' in str(exc_info.value) + + +def test_vsp_query_rejects_band_errors_over_limit(monkeypatch): + class DummyWCS: + def pixel_to_world_values(self, x_pixel, y_pixel): + return 10.0, 20.0 + + def world_to_pixel_values(self, ra_deg, dec_deg): + return np.array([40.0]), np.array([50.0]) + + payload = { + 'chartid': 'X123', + 'photometry': [ + { + 'auid': 'HIGH', + 'ra': '00:00:00.0', + 'dec': '+00:00:00.0', + 'bands': [{'band': 'V', 'mag': 12.0, 'error': 0.051}], + }, + { + 'auid': 'LOW', + 'ra': '00:00:00.0', + 'dec': '+00:00:00.0', + 'bands': [{'band': 'V', 'mag': 12.1, 'error': 0.05}], + }, + ], + } + user_comp_stars = [] + + monkeypatch.setattr(exotic_module, 'search_wcs', lambda file: DummyWCS()) + monkeypatch.setattr(exotic_module, 'radec_hours_to_degree', lambda ra, dec: (10.0, 20.0)) + monkeypatch.setattr( + exotic_module.requests, + 'get', + lambda url, timeout: DummyResponse(payload), + ) + monkeypatch.setattr(exotic_module, 'log_info', lambda *args, **kwargs: None) + + vsp_comp_stars, chart_id = exotic_module.vsp_query( + 'frame.fits', + [100, 100], + 'MObs CV', + 1.0, + user_comp_stars=user_comp_stars, + user_targ_star=[10, 10], + ) + + assert chart_id == 'X123' + assert list(vsp_comp_stars) == ['LOW'] + assert vsp_comp_stars['LOW']['error'] == pytest.approx(0.05) + assert user_comp_stars == [[40, 50]] + + +def test_vsp_query_keeps_late_supplied_matches_after_new_star_limit(monkeypatch): + class DummyWCS: + def pixel_to_world_values(self, x_pixel, y_pixel): + return 10.0, 20.0 + + def world_to_pixel_values(self, ra_deg, dec_deg): + return np.array([ra_deg]), np.array([dec_deg]) + + payload = { + 'chartid': 'X-LIMIT', + 'photometry': [ + { + 'auid': 'NEW-1', + 'ra': '20', + 'dec': '20', + 'bands': [{'band': 'V', 'mag': 11.0, 'error': 0.01}], + }, + { + 'auid': 'NEW-2', + 'ra': '40', + 'dec': '40', + 'bands': [{'band': 'V', 'mag': 12.0, 'error': 0.01}], + }, + { + 'auid': 'SUPPLIED', + 'ra': '80', + 'dec': '80', + 'bands': [{'band': 'V', 'mag': 13.0, 'error': 0.02}], + }, + ], + } + user_comp_stars = [[80, 80]] + + monkeypatch.setattr(exotic_module, 'search_wcs', lambda file: DummyWCS()) + monkeypatch.setattr( + exotic_module, + 'radec_hours_to_degree', + lambda ra, dec: (float(ra), float(dec)), + ) + monkeypatch.setattr( + exotic_module.requests, + 'get', + lambda url, timeout: DummyResponse(payload), + ) + monkeypatch.setattr(exotic_module, 'log_info', lambda *args, **kwargs: None) + + vsp_comp_stars, chart_id = exotic_module.vsp_query( + 'frame.fits', + [100, 100], + 'Clear', + 1.0, + user_comp_stars=user_comp_stars, + max_new_comp_stars=1, + ) + + assert chart_id == 'X-LIMIT' + assert list(vsp_comp_stars) == ['NEW-1', 'SUPPLIED'] + assert vsp_comp_stars['SUPPLIED']['pos'] == [80, 80] + assert user_comp_stars == [[80, 80], [20, 20]] + + +def test_vsp_query_assigns_only_nearest_catalog_source_to_supplied_coordinate(monkeypatch): + class DummyWCS: + def pixel_to_world_values(self, x_pixel, y_pixel): + return 10.0, 20.0 + + def world_to_pixel_values(self, ra_deg, dec_deg): + return np.array([ra_deg]), np.array([dec_deg]) + + payload = { + 'chartid': 'X-NEAREST', + 'photometry': [ + { + 'auid': 'FARTHER', + 'ra': '47', + 'dec': '50', + 'bands': [{'band': 'V', 'mag': 11.0, 'error': 0.01}], + }, + { + 'auid': 'NEAREST', + 'ra': '51', + 'dec': '50', + 'bands': [{'band': 'V', 'mag': 12.0, 'error': 0.02}], + }, + ], + } + user_comp_stars = [[50, 50]] + + monkeypatch.setattr(exotic_module, 'search_wcs', lambda file: DummyWCS()) + monkeypatch.setattr( + exotic_module, + 'radec_hours_to_degree', + lambda ra, dec: (float(ra), float(dec)), + ) + monkeypatch.setattr( + exotic_module.requests, + 'get', + lambda url, timeout: DummyResponse(payload), + ) + monkeypatch.setattr(exotic_module, 'log_info', lambda *args, **kwargs: None) + + vsp_comp_stars, _ = exotic_module.vsp_query( + 'frame.fits', + [100, 100], + 'Clear', + 1.0, + user_comp_stars=user_comp_stars, + max_new_comp_stars=0, + ) + + assert list(vsp_comp_stars) == ['NEAREST'] + assert vsp_comp_stars['NEAREST']['pos'] == [50, 50] + assert user_comp_stars == [[50, 50]] + + +def test_tracked_comparison_position_keeps_full_field_anchor_index_after_science_reset(): + science_comp_stars = [[217.0, 210.0], [408.0, 261.0]] + tracked_calibration_stars = [ + *science_comp_stars, + [415.0, 203.0], + [449.0, 267.0], + ] + + # The target fit restores the shorter science list, while catalog-anchor + # indices retain the full tracking-list index space. + restored_science_comp_stars = list(science_comp_stars) + assert len(restored_science_comp_stars) == 2 + assert exotic_module.tracked_comparison_position( + tracked_calibration_stars, + 2, + ) == [415.0, 203.0] + assert exotic_module.tracked_comparison_position( + tracked_calibration_stars, + 3, + ) == [449.0, 267.0] + + +def test_selected_comparison_finder_entries_include_every_ensemble_member(): + entries = exotic_module.selected_comparison_finder_entries( + [[100.0, 200.0], [300.0, 400.0], [500.0, 600.0]], + ensemble_member_keys=['comp3', 'comp1'], + ) + + assert entries == [ + {'key': 'comp3', 'label': 'Comp 3', 'position': [500.0, 600.0]}, + {'key': 'comp1', 'label': 'Comp 1', 'position': [100.0, 200.0]}, + ] + + +def test_selected_comparison_finder_entries_do_not_depend_on_aavso_metadata(): + entries = exotic_module.selected_comparison_finder_entries( + [[217.0, 210.0]], + comp_index=0, + ) + + assert entries == [ + {'key': 'comp1', 'label': 'Comp 1', 'position': [217.0, 210.0]}, + ] + + +def test_clear_v_calibration_fallback_merges_aavso_with_existing_pool(monkeypatch): + calls = [] + supplied_positions = [[100, 200]] + + def fake_vsp_query(file, axis, obs_filter, img_scale, **kwargs): + calls.append((file, axis, obs_filter, img_scale, kwargs)) + kwargs['user_comp_stars'].append([300, 400]) + return { + '000-BPW-929': { + 'pos': [100, 200], + 'mag': 11.85, + 'error': 0.046, + 'mag_band': 'V', + 'catalog_source': 'AAVSO VSP', + 'is_aavso_vsp': True, + }, + '000-BMX-191': { + 'pos': [300, 400], + 'mag': 12.121, + 'error': 0.005, + 'mag_band': 'V', + 'catalog_source': 'AAVSO VSP', + 'is_aavso_vsp': True, + }, + }, 'X42753ZU' + + monkeypatch.setattr(exotic_module, 'vsp_query', fake_vsp_query) + monkeypatch.setattr(exotic_module, 'log_info', lambda *args, **kwargs: None) + + combined, fallback_stars, chart_id, queried = ( + exotic_module.merge_aavso_vsp_v_calibration_fallback( + 'frame.fits', + [512, 512], + 'Clear', + 1.2, + { + 'NextAstro-g-only': { + 'pos': [100, 200], + 'mag': 11.7, + 'error': 0.01, + 'mag_band': 'g', + 'catalog_source': 'NextAstro photometry catalog', + }, + }, + supplied_positions, + user_targ_star=[250, 250], + ) + ) + + assert queried is True + assert chart_id == 'X42753ZU' + assert len(calls) == 1 + assert calls[0][2] == 'Clear' + assert calls[0][4]['max_new_comp_stars'] == 5 + assert supplied_positions == [[100, 200], [300, 400]] + assert set(fallback_stars) == {'000-BPW-929', '000-BMX-191'} + assert set(combined) == {'NextAstro-g-only', '000-BPW-929', '000-BMX-191'} + + +def test_clear_v_calibration_fallback_skips_vsp_when_nextastro_has_usable_v(monkeypatch): + def unexpected_vsp_query(*args, **kwargs): + raise AssertionError('VSP must not be queried when NextAstro supplied usable V') + + monkeypatch.setattr(exotic_module, 'vsp_query', unexpected_vsp_query) + + existing = { + 'NextAstro-123': { + 'pos': [100, 200], + 'mag': 11.7, + 'error': 0.02, + 'mag_band': 'V', + 'catalog_source': 'NextAstro photometry catalog', + }, + } + combined, fallback_stars, chart_id, queried = ( + exotic_module.merge_aavso_vsp_v_calibration_fallback( + 'frame.fits', + [512, 512], + 'Clear', + 1.2, + existing, + [[100, 200]], + ) + ) + + assert combined == existing + assert fallback_stars == {} + assert chart_id is None + assert queried is False + + +def test_clear_v_calibration_fallback_does_not_repeat_exhausted_vsp_query(monkeypatch): + def unexpected_vsp_query(*args, **kwargs): + raise AssertionError('An exhausted VSP request must not start another retry cycle') + + log_messages = [] + monkeypatch.setattr(exotic_module, 'vsp_query', unexpected_vsp_query) + monkeypatch.setattr( + exotic_module, + 'log_info', + lambda message, **kwargs: log_messages.append((message, kwargs)), + ) + + combined, fallback_stars, chart_id, queried = ( + exotic_module.merge_aavso_vsp_v_calibration_fallback( + 'frame.fits', + [512, 512], + 'Clear', + 1.2, + {}, + [[100, 200]], + vsp_query_available=False, + ) + ) + + assert combined == {} + assert fallback_stars == {} + assert chart_id is None + assert queried is False + assert 'already exhausted all retries' in log_messages[-1][0] + assert log_messages[-1][1].get('warn') is True + + +@pytest.mark.parametrize( + 'observed_filter', + ['CV', 'Clear', 'Luminance', 'Photographic G', 'Gaia G'], +) +def test_build_stellar_variability_params_records_nextastro_reference( + monkeypatch, tmp_path, observed_filter): + captured = {} + + class DummyFit: + data = np.array([1.0, 1.02, 0.98], dtype=float) + dataerr = np.full(3, 0.01, dtype=float) + airmass_model = np.ones(3, dtype=float) + airmass = np.array([1.1, 1.2, 1.3], dtype=float) + jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) + transit = np.ones(3, dtype=float) + stellar_variability_target_flux = np.array([1000.0, 1020.0, 980.0], dtype=float) + stellar_variability_comp_flux = np.full(3, 1000.0, dtype=float) + stellar_variability_target_flux_error = np.full(3, 2.0, dtype=float) + stellar_variability_comp_flux_error = np.full(3, 2.0, dtype=float) + + def fake_plot(params, save, s_name, label): + captured['params'] = params + captured['label'] = label + + monkeypatch.setattr(exotic_module, 'plot_stellar_variability', fake_plot) + + calibration_star = { + 'mag': 12.0, + 'error': 0.05, + 'ra': 10.1, + 'dec': -20.2, + 'catalog_ra': 10.10001, + 'catalog_dec': -20.20001, + 'catalog_source': 'NextAstro photometry catalog', + 'is_aavso_vsp': False, + 'mag_band': 'V', + 'observed_filter': 'V', + 'source_id': 123, + 'separation_arcsec': 0.2, + } + + params = exotic_module.build_stellar_variability_params_from_fit( + DummyFit(), + calibration_star, + [100, 200], + 'NextAstro-123', + tmp_path, + 'Host Star', + observed_filter=observed_filter, + ) + + assert captured['label'] == 'RA=10.1000000 Dec=-20.2000000' + assert len(params) == 3 + assert params[0]['catalog_source'] == 'NextAstro photometry catalog' + assert params[0]['is_aavso_vsp'] is False + assert params[0]['comp_ra'] == pytest.approx(10.1) + assert params[0]['comp_dec'] == pytest.approx(-20.2) + assert params[0]['cmag'] == pytest.approx(12.0) + assert params[0]['cmag_err'] == pytest.approx(0.05) + assert params[0]['observed_filter'] == observed_filter + assert params[0]['mag_band'] == 'ClearV' + assert params[0]['catalog_mag_band'] == 'V' + + +def test_build_stellar_variability_params_rejects_cross_band_calibration(tmp_path): + class DummyFit: + data = np.ones(3, dtype=float) + dataerr = np.full(3, 0.01, dtype=float) + airmass_model = np.ones(3, dtype=float) + airmass = np.array([1.1, 1.2, 1.3], dtype=float) + jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) + transit = np.ones(3, dtype=float) + stellar_variability_target_flux = np.full(3, 1000.0, dtype=float) + stellar_variability_comp_flux = np.full(3, 1000.0, dtype=float) + stellar_variability_target_flux_error = np.full(3, 2.0, dtype=float) + stellar_variability_comp_flux_error = np.full(3, 2.0, dtype=float) + + with pytest.raises(RuntimeError, match='cross-band absolute calibration is not permitted'): + exotic_module.build_stellar_variability_params_from_fit( + DummyFit(), + { + 'mag': 11.615, + 'error': 0.001, + 'mag_band': 'g', + 'observed_filter': 'V', + }, + [100, 200], + 'NextAstro-invalid-g-reference', + tmp_path, + 'Host Star', + observed_filter='V', + observation_date='2026-08-02', + ) + + differential_csv = next( + tmp_path.glob('StellarVariabilityDifferentialMagnitude_HostStar_2026-08-02.csv') + ) + assert '# AIRMASS_CORRECTION=NO' in differential_csv.read_text(encoding='utf-8') + assert ( + tmp_path / 'StellarVariabilityDifferentialMagnitude_HostStar_2026-08-02.png' + ).exists() + + +def test_build_stellar_variability_params_uses_raw_ratio_and_per_exposure_errors(monkeypatch, tmp_path): + comp_mag = 9.751 + comp_mag_error = 0.018 + target_mag = 13.1 + flux_ratio = 10 ** ((comp_mag - target_mag) / 2.5) + detrended = flux_ratio * np.array([0.94, 1.0, 1.06], dtype=float) + + class DummyFit: + # The fitted series is intentionally normalized: the absolute target + # magnitude must come from the retained raw target/comparison fluxes. + data = detrended / np.nanmedian(detrended) + dataerr = np.full(3, 0.01, dtype=float) + airmass_model = np.array([0.94, 1.0, 1.06], dtype=float) + airmass = np.array([1.1, 1.2, 1.3], dtype=float) + jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) + transit = np.ones(3, dtype=float) + stellar_variability_comp_flux = np.full(3, 100000.0, dtype=float) + stellar_variability_target_flux = stellar_variability_comp_flux * detrended + stellar_variability_target_flux_error = np.array([20.0, 21.0, 22.0], dtype=float) + stellar_variability_comp_flux_error = np.array([30.0, 31.0, 32.0], dtype=float) + + monkeypatch.setattr(exotic_module, 'plot_stellar_variability', lambda *args, **kwargs: None) + + calibration_star = { + 'mag': comp_mag, + 'error': comp_mag_error, + 'catalog_source': 'AAVSO VSP', + 'is_aavso_vsp': True, + 'mag_band': 'V', + 'observed_filter': 'V', + } + + params = exotic_module.build_stellar_variability_params_from_fit( + DummyFit(), + calibration_star, + [100, 200], + '000-BJX-718', + tmp_path, + 'HAT-P-37', + observed_filter='CV', + ) + + expected_mag_error = np.hypot( + comp_mag_error, + (2.5 / np.log(10.0)) * np.hypot( + DummyFit.stellar_variability_target_flux_error[1] + / DummyFit.stellar_variability_target_flux[1], + DummyFit.stellar_variability_comp_flux_error[1] + / DummyFit.stellar_variability_comp_flux[1], + ), + ) + + expected_raw_magnitudes = comp_mag - (2.5 * np.log10(detrended)) + np.testing.assert_allclose( + [row['mag'] for row in params], + expected_raw_magnitudes, + atol=1.0e-10, + ) + np.testing.assert_allclose( + [row['differential_mag'] for row in params], + -2.5 * np.log10(detrended), + atol=1.0e-10, + ) + assert params[0]['mag'] != pytest.approx(target_mag) + assert params[1]['mag_err'] == pytest.approx(expected_mag_error) + assert params[1]['differential_mag_err'] == pytest.approx( + np.sqrt(expected_mag_error ** 2 - comp_mag_error ** 2) + ) + assert params[1]['mag_err'] < 0.08 + + +def test_annotate_stellar_variability_raw_photometry_restores_final_selected_fit_fluxes(): + class DummyFit: + data = np.array([0.99, 1.0, 1.01], dtype=float) + + target_flux = np.array([9900.0, 10000.0, 10100.0], dtype=float) + comp_flux = np.full(3, 20000.0, dtype=float) + target_error = np.array([10.0, 11.0, 12.0], dtype=float) + comp_error = np.array([20.0, 21.0, 22.0], dtype=float) + fit = DummyFit() + + exotic_module.annotate_stellar_variability_raw_photometry( + fit, + target_flux, + comp_flux, + target_flux_error=target_error, + comp_flux_error=comp_error, + ) + + retained = exotic_module.stellar_variability_raw_photometry(fit) + for actual, expected in zip( + retained, + (target_flux, comp_flux, target_error, comp_error), + ): + np.testing.assert_array_equal(actual, expected) + + +def test_build_stellar_variability_params_rejects_normalized_only_absolute_calibration( + monkeypatch, tmp_path): + class DummyFit: + data = np.array([0.99, 1.0, 1.01], dtype=float) + dataerr = np.full(3, 0.01, dtype=float) + airmass = np.ones(3, dtype=float) + jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) + transit = np.ones(3, dtype=float) + + monkeypatch.setattr(exotic_module, 'plot_stellar_variability', lambda *args, **kwargs: None) + + with pytest.raises(RuntimeError, match='cannot be recovered from a normalized light curve'): + exotic_module.build_stellar_variability_params_from_fit( + DummyFit(), + {'mag': 12.0, 'error': 0.02, 'mag_band': 'V'}, + [100, 200], + 'COMP', + tmp_path, + 'Host Star', + observed_filter='V', + ) + + +def test_stellar_variability_requires_selected_transit_comparison(monkeypatch, tmp_path): + logged = [] + + class DummyFit: + data = np.array([1.0, 1.01, 0.99], dtype=float) + airmass_model = np.ones(3, dtype=float) + airmass = np.ones(3, dtype=float) + jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) + transit = np.ones(3, dtype=float) + + monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) + + params = exotic_module.stellar_variability( + {0: {'myfit': DummyFit(), 'pos': [100, 200]}}, + DummyFit(), + [[100, 200]], + {'REF': {'pos': [100, 200], 'mag': 12.0, 'error': 0.02}}, + [0], + None, + tmp_path, + 'Host Star', + ) + + assert params == [] + assert any('no transit-fit comparison star' in message for message in logged) + + +def test_stellar_variability_derives_selected_comparison_catalog_magnitude(monkeypatch, tmp_path): + logged = [] + captured = {} + + class DummyFit: + def __init__(self, data): + self.data = np.array(data, dtype=float) + self.dataerr = np.full(3, 0.01, dtype=float) + self.airmass_model = np.ones(3, dtype=float) + self.airmass = np.ones(3, dtype=float) + self.jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) + self.transit = np.ones(3, dtype=float) + self.stellar_variability_target_flux = self.data * 1000.0 + self.stellar_variability_comp_flux = np.full(3, 1000.0, dtype=float) + self.stellar_variability_target_flux_error = np.full(3, 2.0, dtype=float) + self.stellar_variability_comp_flux_error = np.full(3, 2.0, dtype=float) + + monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) + monkeypatch.setattr( + exotic_module, + 'plot_stellar_variability', + lambda params, save, s_name, label: captured.update(params=params, label=label), + ) + + selected_mag = 11.0 + anchor_mag = 12.0 + selected_to_anchor_flux_ratio = 10 ** ((anchor_mag - selected_mag) / 2.5) + selected_fit = DummyFit([1.0, 1.01, 0.99]) + anchor_fit = DummyFit(selected_to_anchor_flux_ratio * np.array([1.0, 1.01, 0.99])) + + params = exotic_module.stellar_variability( + { + 0: {'myfit': selected_fit, 'pos': [100, 200]}, + 1: {'myfit': anchor_fit, 'pos': [300, 400]}, + }, + DummyFit([1.0, 1.01, 0.99]), + [[100, 200], [300, 400]], + {'REF': {'pos': [300, 400], 'mag': anchor_mag, 'error': 0.02}}, + [1], + 0, + tmp_path, + 'Host Star', + comp_ra_dec=[(10.0, -20.0), (11.0, -21.0)], + ) + + assert len(params) == 3 + assert params[0]['cmag'] == pytest.approx(selected_mag) + assert params[0]['cmag_err'] == pytest.approx(0.02) + assert params[0]['comp_ra'] == pytest.approx(10.0) + assert params[0]['comp_dec'] == pytest.approx(-20.0) + assert params[0]['derived_catalog_reference'] is True + assert params[0]['derived_reference_anchor_count'] == 1 + assert params[0]['derived_reference_anchor_labels'] == ['REF'] + assert captured['label'] == 'RA=10.0000000 Dec=-20.0000000' + assert any('derived catalog magnitude' in message for message in logged) + + +def test_stellar_variability_uses_direct_catalog_without_shared_fit_oot_points(monkeypatch, tmp_path): + logged = [] + + class DummyFit: + def __init__(self, data): + self.data = np.array(data, dtype=float) + self.dataerr = np.full(4, 0.01, dtype=float) + self.airmass_model = np.ones(4, dtype=float) + self.airmass = np.ones(4, dtype=float) + self.jd_times = np.array([2450000.1, 2450000.2, 2450000.3, 2450000.4], dtype=float) + self.transit = np.ones(4, dtype=float) + self.stellar_variability_target_flux = self.data * 1000.0 + self.stellar_variability_comp_flux = np.full(4, 1000.0, dtype=float) + self.stellar_variability_target_flux_error = np.full(4, 2.0, dtype=float) + self.stellar_variability_comp_flux_error = np.full(4, 2.0, dtype=float) + + monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) + monkeypatch.setattr(exotic_module, 'plot_stellar_variability', lambda *args, **kwargs: None) + + selected_fit = DummyFit([1.0, 1.0, 1.0, 1.0]) + noisy_anchor_fit = DummyFit([2.0, 0.8, 2.2, 0.7]) + best_fit = DummyFit([1.0, 1.0, 1.0, 1.0]) + best_fit.transit = np.zeros(4, dtype=float) + params = exotic_module.stellar_variability( + { + 0: {'myfit': selected_fit, 'pos': [100, 200]}, + 1: {'myfit': noisy_anchor_fit, 'pos': [300, 400]}, + }, + best_fit, + [[100, 200], [300, 400]], + {'ANCHOR': {'pos': [300, 400], 'mag': 12.0, 'error': 0.02, 'mag_band': 'g'}}, + [1], + 0, + tmp_path, + 'Host Star', + observed_filter='g', + comp_ra_dec=[(10.0, -20.0), (11.0, -21.0)], + field_catalog={ + 'rows': [{ + 'ra': 10.0, + 'dec': -20.0, + 'g': 11.5, + 'dg': 0.08, + 'source_id': 12345, + }] + }, + ) + + assert len(params) == 4 + assert params[0]['cmag'] == pytest.approx(11.5) + assert params[0]['cmag_err'] == pytest.approx(0.08) + assert params[0]['derived_catalog_reference'] is False + assert params[0]['allow_high_error_catalog_reference'] is True + assert any('direct selected-comparison catalog magnitude' in message for message in logged) + + +def test_stellar_variability_derives_catalog_magnitude_from_full_field(monkeypatch, tmp_path): + class DummyFit: + data = np.array([1.0, 1.01, 0.99], dtype=float) + dataerr = np.full(3, 0.01, dtype=float) + airmass_model = np.ones(3, dtype=float) + airmass = np.ones(3, dtype=float) + jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) + transit = np.ones(3, dtype=float) + stellar_variability_target_flux = data * 1000.0 + stellar_variability_comp_flux = np.full(3, 1000.0, dtype=float) + stellar_variability_target_flux_error = np.full(3, 2.0, dtype=float) + stellar_variability_comp_flux_error = np.full(3, 2.0, dtype=float) + + class DummyWcs: + def world_to_pixel_values(self, ra, dec): + return float(ra), float(dec) + + def pixel_to_world_values(self, x, y): + return 123.4, -45.6 + + image = np.full((60, 60), 10.0, dtype=float) + image[20, 20] = 50.0 + image[40, 40] = 110.0 + + monkeypatch.setattr(exotic_module, 'plot_stellar_variability', lambda *args, **kwargs: None) + monkeypatch.setattr(exotic_module, 'search_wcs', lambda _path: DummyWcs()) + + params = exotic_module.stellar_variability( + {0: {'myfit': DummyFit(), 'pos': [20, 20]}}, + DummyFit(), + [[20, 20]], + {}, + [], + 0, + tmp_path, + 'Host Star', + observed_filter='g', + field_catalog={ + 'rows': [{ + 'ra': 40.0, + 'dec': 40.0, + 'g': 12.0, + 'dg': 0.03, + 'source_id': 67890, + }] + }, + reference_image=image, + wcs_file='dummy.wcs', + ) + + expected_mag = 12.0 - 2.5 * np.log10(40.0 / 100.0) + assert len(params) == 3 + assert params[0]['cmag'] == pytest.approx(expected_mag) + assert params[0]['cmag_err'] == pytest.approx(0.03) + assert params[0]['comp_ra'] == pytest.approx(123.4) + assert params[0]['comp_dec'] == pytest.approx(-45.6) + assert params[0]['derived_catalog_reference'] is True + assert params[0]['derived_reference_anchor_count'] == 1 + assert params[0]['derived_reference_anchor_labels'] == ['NextAstro-67890'] + + +def test_derived_catalog_reference_skips_missing_anchor_fit(): + class DummyFit: + data = np.array([1.0, 1.01, 0.99], dtype=float) + + label, star = exotic_module.derived_catalog_reference_for_selected_comp( + { + 0: {'myfit': DummyFit(), 'pos': [10, 10]}, + 1: None, + }, + [[10, 10], [20, 20]], + { + 'Anchor': { + 'pos': [20, 20], + 'mag': 12.0, + 'error': 0.03, + 'mag_band': 'V', + }, + }, + [1], + 0, + observed_filter='V', + ) + + assert label is None + assert star is None + + +def test_derived_catalog_reference_ensembles_multiple_aavso_v_anchors(): + class DummyFit: + def __init__(self, reference_curve): + reference_curve = np.asarray(reference_curve, dtype=float) + self.data = reference_curve + self.time = np.array([1.0, 2.0, 3.0], dtype=float) + self.transit = np.ones(3, dtype=float) + self.stellar_variability_target_flux = reference_curve * 1000.0 + self.stellar_variability_comp_flux = np.full(3, 1000.0, dtype=float) + self.stellar_variability_target_flux_error = np.full(3, 2.0, dtype=float) + self.stellar_variability_comp_flux_error = np.full(3, 2.0, dtype=float) + + selected_mag = 11.0 + first_anchor_mag = 12.0 + second_anchor_mag = 13.0 + first_ratio = 10.0 ** ((first_anchor_mag - selected_mag) / 2.5) + second_ratio = 10.0 ** ((second_anchor_mag - selected_mag) / 2.5) + + label, star = exotic_module.derived_catalog_reference_for_selected_comp( + { + 0: {'myfit': DummyFit(np.ones(3)), 'pos': [10, 10]}, + 1: {'myfit': DummyFit(np.full(3, first_ratio)), 'pos': [20, 20]}, + 2: {'myfit': DummyFit(np.full(3, second_ratio)), 'pos': [30, 30]}, + }, + [[10, 10], [20, 20], [30, 30]], + { + 'AAVSO-1': { + 'pos': [20, 20], + 'mag': first_anchor_mag, + 'error': 0.02, + 'mag_band': 'V', + 'catalog_source': 'AAVSO VSP', + 'is_aavso_vsp': True, + }, + 'AAVSO-2': { + 'pos': [30, 30], + 'mag': second_anchor_mag, + 'error': 0.04, + 'mag_band': 'V', + 'catalog_source': 'AAVSO VSP', + 'is_aavso_vsp': True, + }, + }, + [1, 2], + 0, + observed_filter='Clear', + ) + + assert label == 'Derived Comp 1' + assert star['mag'] == pytest.approx(selected_mag) + assert star['error'] == pytest.approx((1.0 / (1.0 / 0.02 ** 2 + 1.0 / 0.04 ** 2)) ** 0.5) + assert star['mag_band'] == 'V' + assert star['derived_catalog_reference'] is True + assert star['derived_reference_anchor_count'] == 2 + assert star['derived_reference_anchor_labels'] == ['AAVSO-1', 'AAVSO-2'] + + +def test_stellar_variability_rejects_g_catalog_anchor_for_clearv(monkeypatch, tmp_path): + logged = [] + + class DummyFit: + data = np.array([1.0, 1.01, 0.99], dtype=float) + dataerr = np.full(3, 0.01, dtype=float) + airmass_model = np.ones(3, dtype=float) + airmass = np.ones(3, dtype=float) + jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) + transit = np.ones(3, dtype=float) + stellar_variability_target_flux = data * 1000.0 + stellar_variability_comp_flux = np.full(3, 1000.0, dtype=float) + stellar_variability_target_flux_error = np.full(3, 2.0, dtype=float) + stellar_variability_comp_flux_error = np.full(3, 2.0, dtype=float) + + class DummyWcs: + def world_to_pixel_values(self, ra, dec): + return float(ra), float(dec) + + def pixel_to_world_values(self, x, y): + return 123.4, -45.6 + + image = np.full((60, 60), 10.0, dtype=float) + image[20, 20] = 50.0 + image[40, 40] = 110.0 + + monkeypatch.setattr(exotic_module, 'plot_stellar_variability', lambda *args, **kwargs: None) + monkeypatch.setattr(exotic_module, 'search_wcs', lambda _path: DummyWcs()) + monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) + + params = exotic_module.stellar_variability( + {0: {'myfit': DummyFit(), 'pos': [20, 20]}}, + DummyFit(), + [[20, 20]], + {}, + [], + 0, + tmp_path, + 'Host Star', + observed_filter='CV', + field_catalog={ + 'rows': [{ + 'ra': 40.0, + 'dec': 40.0, + 'g': 12.0, + 'dg': 0.03, + 'source_id': 67890, + }] + }, + reference_image=image, + wcs_file='dummy.wcs', + ) + + assert params == [] + assert any('no derived magnitude could be inferred' in message for message in logged) + + +def test_stellar_variability_uses_v_catalog_anchor_for_clearv(monkeypatch, tmp_path): + class DummyFit: + data = np.array([1.0, 1.01, 0.99], dtype=float) + dataerr = np.full(3, 0.01, dtype=float) + airmass_model = np.ones(3, dtype=float) + airmass = np.ones(3, dtype=float) + jd_times = np.array([2450000.1, 2450000.2, 2450000.3], dtype=float) + transit = np.ones(3, dtype=float) + stellar_variability_target_flux = data * 1000.0 + stellar_variability_comp_flux = np.full(3, 1000.0, dtype=float) + stellar_variability_target_flux_error = np.full(3, 2.0, dtype=float) + stellar_variability_comp_flux_error = np.full(3, 2.0, dtype=float) + + class DummyWcs: + def world_to_pixel_values(self, ra, dec): + return float(ra), float(dec) + + def pixel_to_world_values(self, x, y): + return 123.4, -45.6 + + image = np.full((60, 60), 10.0, dtype=float) + image[20, 20] = 50.0 + image[40, 40] = 110.0 + + monkeypatch.setattr(exotic_module, 'plot_stellar_variability', lambda *args, **kwargs: None) + monkeypatch.setattr(exotic_module, 'search_wcs', lambda _path: DummyWcs()) + + params = exotic_module.stellar_variability( + {0: {'myfit': DummyFit(), 'pos': [20, 20]}}, + DummyFit(), + [[20, 20]], + {}, + [], + 0, + tmp_path, + 'Host Star', + observed_filter='CV', + field_catalog={ + 'rows': [{ + 'ra': 40.0, + 'dec': 40.0, + 'Vmag': 12.0, + 'err_Vmag': 0.03, + 'g': 11.7, + 'dg': 0.01, + 'source_id': 67890, + }] + }, + reference_image=image, + wcs_file='dummy.wcs', + ) + + assert len(params) == 3 + assert params[0]['mag_band'] == 'ClearV' + assert params[0]['catalog_mag_band'] == 'V' + assert params[0]['cmag_err'] == pytest.approx(0.03) + + +def test_check_for_variable_stars_uses_nextastro_flags_to_filter(monkeypatch): + logged = [] + + ra_wcs = np.array([[100.1, 100.2], [100.3, 100.4]]) + dec_wcs = np.array([[-10.1, -10.2], [-10.3, -10.4]]) + comp_stars = [[0, 0], [1, 1]] + + monkeypatch.setattr(exotic_module, 'nextastro_variability_test', lambda payload: [False, True]) + monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) + + exotic_module.check_for_variable_stars( + ra_wcs, dec_wcs, comp_stars, use_nextastro_variability_server=True + ) + + assert comp_stars == [[0, 0]] + assert any('NextAstro flagged variable: False' in message for message in logged) + assert any('NextAstro flagged variable: True' in message for message in logged) + + +def test_check_for_variable_stars_logs_underlying_nextastro_retry_error(monkeypatch): + logged = [] + + ra_wcs = np.array([[100.1]]) + dec_wcs = np.array([[-10.1]]) + comp_stars = [[0, 0]] + + last_attempt = Future(5) + last_attempt.set_exception(RuntimeError('HTTP 502')) + + def raise_retry_error(payload): + raise RetryError(last_attempt) + + monkeypatch.setattr(exotic_module, 'nextastro_variability_test', raise_retry_error) + monkeypatch.setattr(exotic_module, 'query_variable_star_apis', lambda ra, dec: False) + monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) + + exotic_module.check_for_variable_stars( + ra_wcs, dec_wcs, comp_stars, use_nextastro_variability_server=True + ) + + assert any('RetryError after 5 attempts (RuntimeError: HTTP 502)' in message for message in logged) + + +def test_get_wcs_falls_back_to_nextastro_when_nova_fails(monkeypatch): + service_calls = [] + + class DummyPlateSolution: + def __init__(self, **kwargs): + pass + + def plate_solution(self): + service_calls.append('nova') + return False + + class DummyNextAstroSolution: + def __init__(self, **kwargs): + pass + + def plate_solution(self): + service_calls.append('nextastro') + return 'nextastro-wcs' + + monkeypatch.setattr(exotic_module, 'PlateSolution', DummyPlateSolution) + monkeypatch.setattr(exotic_module, 'NextAstroPlateSolution', DummyNextAstroSolution) + monkeypatch.setattr(exotic_module, 'animate_toggle', lambda *args, **kwargs: None) + + solved_wcs = exotic_module.get_wcs('frame.fits', directory='.') + + assert solved_wcs == 'nextastro-wcs' + assert service_calls == ['nova', 'nextastro'] + + +def test_get_wcs_logs_nextastro_bad_gateway_before_nova_fallback(monkeypatch): + logged = [] + service_calls = [] + + class DummyPlateSolution: + def __init__(self, **kwargs): + self.last_error_type = None + + def plate_solution(self): + service_calls.append('nova') + return 'nova-wcs' + + class DummyNextAstroSolution: + def __init__(self, **kwargs): + self.last_http_status = 502 + self.last_error_type = 'NextAstro solve submission' + self.api_url = 'https://astrometry.nextastro.org' + + def plate_solution(self): + service_calls.append('nextastro') + return False + + monkeypatch.setattr(exotic_module, 'PlateSolution', DummyPlateSolution) + monkeypatch.setattr(exotic_module, 'NextAstroPlateSolution', DummyNextAstroSolution) + monkeypatch.setattr(exotic_module, 'animate_toggle', lambda *args, **kwargs: None) + monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) + + solved_wcs = exotic_module.get_wcs('frame.fits', directory='.', use_nextastro_astrometry=True) + + assert solved_wcs == 'nova-wcs' + assert service_calls == ['nextastro', 'nova'] + assert any(message == 'NextAstro Server not responding. Will try nova.astrometry.net' for message in logged) + + +def test_get_wcs_logs_nextastro_bad_gateway_after_both_methods_fail(monkeypatch): + logged = [] + service_calls = [] + + class DummyPlateSolution: + def __init__(self, **kwargs): + self.last_error_type = 'Upload' + + def plate_solution(self): + service_calls.append('nova') + return False + + class DummyNextAstroSolution: + def __init__(self, **kwargs): + self.last_http_status = 502 + self.last_error_type = 'NextAstro solve submission' + self.api_url = 'https://astrometry.nextastro.org' + + def plate_solution(self): + service_calls.append('nextastro') + return False + + monkeypatch.setattr(exotic_module, 'PlateSolution', DummyPlateSolution) + monkeypatch.setattr(exotic_module, 'NextAstroPlateSolution', DummyNextAstroSolution) + monkeypatch.setattr(exotic_module, 'animate_toggle', lambda *args, **kwargs: None) + monkeypatch.setattr(exotic_module, 'log_info', lambda message, warn=False, error=False: logged.append(message)) + + solved_wcs = exotic_module.get_wcs('frame.fits', directory='.') + + assert solved_wcs is False + assert service_calls == ['nova', 'nextastro'] + assert any(message == 'NextAstro Server not responding. Both astrometry methods trialed, pushing forward without astrometry solution' + for message in logged) + + +def test_vsx_variable_falls_back_to_nextastro(monkeypatch): + class DummyFailedResponse: + def raise_for_status(self): + raise RuntimeError('VSX unavailable') + + monkeypatch.setattr(exotic_module.requests, 'get', lambda *args, **kwargs: DummyFailedResponse()) + monkeypatch.setattr(exotic_module, 'nextastro_variability_test', lambda payload: [True]) + + is_variable = exotic_module.vsx_variable(ra=10.0, dec=20.0) + + assert is_variable is True + + +def test_vsx_variable_handles_empty_default_response_without_fallback(monkeypatch): + nextastro_called = {'value': False} + + monkeypatch.setattr( + exotic_module.requests, + 'get', + lambda *args, **kwargs: DummyResponse({'VSXObjects': []}), + ) + monkeypatch.setattr( + exotic_module, + 'nextastro_variability_test', + lambda payload: nextastro_called.__setitem__('value', True), + ) + + is_variable = exotic_module.vsx_variable(ra=10.0, dec=20.0) + + assert is_variable is False + assert nextastro_called['value'] is False + + +def test_vsx_variable_parses_default_vsx_object_list(monkeypatch): + monkeypatch.setattr( + exotic_module.requests, + 'get', + lambda *args, **kwargs: DummyResponse({ + 'VSXObjects': { + 'VSXObject': [ + { + 'Name': 'alf Ori', + 'RA2000': '88.79292', + 'Declination2000': '7.40706', + 'Category': 'Variable', + } + ] + } + }), + ) + monkeypatch.setattr(exotic_module, 'nextastro_variability_test', lambda payload: [False]) + + is_variable = exotic_module.vsx_variable(ra=88.79292, dec=7.40706) + + assert is_variable is True + + +def _stellar_variability_only_planet_dict(): + return { + 'pName': 'Synthetic b', + 'sName': 'Synthetic', + 'pPer': 1.0, + 'pPerUnc': 0.0001, + 'midT': 10.0, + 'midTUnc': 0.0001, + 'rprs': 0.1, + 'rprsUnc': 0.001, + 'aRs': 12.0, + 'aRsUnc': 0.2, + 'inc': 89.0, + 'incUnc': 0.1, + 'ecc': 0.0, + 'omega': 90.0, + } + + +def test_stellar_variability_out_of_transit_mask_excludes_predicted_transit_window(): + p_dict = _stellar_variability_only_planet_dict() + duration = exotic_module.estimate_transit_duration_from_prior_geometry( + exotic_module.stellar_variability_transit_prior_from_planet_dict(p_dict) + ) + times = np.array([ + p_dict['midT'] - duration, + p_dict['midT'], + p_dict['midT'] + 0.49 * duration, + p_dict['midT'] + duration, + ]) + + keep_mask, summary = exotic_module.stellar_variability_out_of_transit_mask(times, p_dict) + + assert keep_mask.tolist() == [True, False, False, True] + assert summary['applied'] is True + assert summary['rejected_point_count'] == 2 + assert summary['duration_days'] == pytest.approx(duration) + + +def test_build_stellar_variability_only_lightcurve_discards_transit_points(monkeypatch): + p_dict = _stellar_variability_only_planet_dict() + duration = exotic_module.estimate_transit_duration_from_prior_geometry( + exotic_module.stellar_variability_transit_prior_from_planet_dict(p_dict) + ) + offsets = np.array([-3.0, -2.2, -1.4, -0.7, -0.1, 0.0, 0.1, 0.7, 1.4, 2.2, 3.0]) * duration + times = p_dict['midT'] + offsets + target_flux = np.full(times.shape, 10000.0) + comp_flux = np.full(times.shape, 10000.0) + flux_err = np.full(times.shape, 20.0) + airmass = np.ones(times.shape) + + monkeypatch.setattr( + exotic_module, + 'get_phase', + lambda t, per, tmid: ((np.asarray(t, dtype=float) - tmid) / per + 0.5) % 1.0 - 0.5, + ) + + fit, prepared = exotic_module.build_stellar_variability_only_lightcurve_from_fluxes( + times, + target_flux, + comp_flux, + airmass, + p_dict, + jd_times=times, + target_flux_error=flux_err, + comp_flux_error=flux_err, + exposure_times_seconds=np.full(times.shape, 60.0), + gain_e_per_adu=1.0, + comp_index=0, + comp_label="Comp 1", + comp_position=[1, 2], + method_label="PSF photometry", + ) + + assert prepared['applied'] is True + assert fit is not None + assert fit.stellar_variability_only is True + assert np.all(fit.transit == 1.0) + assert fit.stellar_variability_transit_exclusion['rejected_point_count'] == 3 + assert not np.any(np.isclose(fit.time, p_dict['midT'])) + assert len(fit.time) == times.size - 3 + + +def test_stellar_variability_ensemble_masks_saturated_frames_and_error_clips_members(): + frame_count = 8 + ranked_summaries = [ + { + 'key': 'comp1', 'comp_index': 0, 'label': 'Comp 1', 'position': [10, 20], + 'overexposure_rejected_count': 0, + }, + { + 'key': 'comp2', 'comp_index': 1, 'label': 'Comp 2', 'position': [30, 40], + 'overexposure_rejected_count': 0, + }, + { + 'key': 'comp3', 'comp_index': 2, 'label': 'Comp 3', 'position': [50, 60], + 'overexposure_rejected_count': 1, + }, + { + 'key': 'comp4', 'comp_index': 3, 'label': 'Comp 4', 'position': [70, 80], + 'overexposure_rejected_count': 0, + }, + ] + comp_flux_map = { + 'comp1': np.full(frame_count, 1000.0), + 'comp2': np.full(frame_count, 2000.0), + 'comp3': np.full(frame_count, 3000.0), + 'comp4': np.full(frame_count, 1500.0), + } + comp_flux_map['comp3'][0] = np.nan + calibration_stars = { + 'C1': {'pos': [10, 20], 'mag': 12.0, 'error': 0.010, 'mag_band': 'V'}, + 'C2': {'pos': [30, 40], 'mag': 12.5, 'error': 0.011, 'mag_band': 'V'}, + 'C3': {'pos': [50, 60], 'mag': 12.2, 'error': 0.010, 'mag_band': 'V'}, + 'C4': {'pos': [70, 80], 'mag': 12.3, 'error': 0.200, 'mag_band': 'V'}, + } + + selection = exotic_module.select_stellar_variability_ensemble_members( + ranked_summaries, + calibration_stars, + comp_flux_map, + observed_filter='V', + ) + + assert [member['key'] for member in selection['members']] == ['comp3', 'comp2', 'comp1'] + assert selection['members'][0]['summary']['overexposure_rejected_count'] == 1 + rejected_reasons = {item['key']: item['reason'] for item in selection['rejected']} + assert 'sigma-clip' in rejected_reasons['comp4'] + assert selection['calibration_error_clip']['high_threshold'] < 0.2 + assert selection['calibration_error_clip']['high_threshold'] >= 0.01 + assert selection['calibration_error_clip']['minimum_high_threshold'] == pytest.approx(0.01) + + +def test_stellar_variability_ensemble_skips_cross_band_catalog_reference(): + frame_count = 8 + ranked_summaries = [ + { + 'key': 'comp1', 'comp_index': 0, 'label': 'Comp 1', 'position': [10, 20], + 'overexposure_rejected_count': 0, + }, + { + 'key': 'comp2', 'comp_index': 1, 'label': 'Comp 2', 'position': [30, 40], + 'overexposure_rejected_count': 0, + }, + ] + calibration_stars = { + 'C1': {'pos': [10, 20], 'mag': 11.0, 'error': 0.001, 'mag_band': 'g'}, + 'C2': {'pos': [30, 40], 'mag': 12.5, 'error': 0.011, 'mag_band': 'V'}, + } + selection = exotic_module.select_stellar_variability_ensemble_members( + ranked_summaries, + calibration_stars, + { + 'comp1': np.full(frame_count, 2000.0), + 'comp2': np.full(frame_count, 1000.0), + }, + observed_filter='V', + min_members=1, + max_members=1, + ) + + assert [member['key'] for member in selection['members']] == ['comp2'] + assert selection['members'][0]['star']['mag_band'] == 'V' + rejected = {item['key']: item['reason'] for item in selection['rejected']} + assert rejected['comp1'] == 'no usable catalog calibration' + + +def test_stellar_variability_ensemble_combines_nextastro_and_aavso_v_members(): + frame_count = 8 + ranked_summaries = [ + { + 'key': 'comp1', 'comp_index': 0, 'label': 'Comp 1', 'position': [10, 20], + 'overexposure_rejected_count': 0, + }, + { + 'key': 'comp2', 'comp_index': 1, 'label': 'Comp 2', 'position': [30, 40], + 'overexposure_rejected_count': 0, + }, + ] + calibration_stars = { + 'NextAstro-123': { + 'pos': [10, 20], + 'mag': 11.8, + 'error': 0.02, + 'mag_band': 'V', + 'catalog_source': 'NextAstro photometry catalog', + 'source_id': 123, + }, + '000-BMX-191': { + 'pos': [30, 40], + 'mag': 12.121, + 'error': 0.005, + 'mag_band': 'V', + 'catalog_source': 'AAVSO VSP', + 'is_aavso_vsp': True, + 'catalog_ra': 18.0, + 'catalog_dec': 35.0, + }, + } + + selection = exotic_module.select_stellar_variability_ensemble_members( + ranked_summaries, + calibration_stars, + { + 'comp1': np.full(frame_count, 2000.0), + 'comp2': np.full(frame_count, 1000.0), + }, + observed_filter='Clear', + ) + + assert [member['key'] for member in selection['members']] == ['comp1', 'comp2'] + assert [member['label'] for member in selection['members']] == [ + 'NextAstro-123', + '000-BMX-191', + ] + assert [member['star']['catalog_source'] for member in selection['members']] == [ + 'NextAstro photometry catalog', + 'AAVSO VSP', + ] + + +def test_stellar_variability_single_mode_can_select_aavso_v_fallback_member(): + frame_count = 8 + selection = exotic_module.select_stellar_variability_ensemble_members( + [ + { + 'key': 'comp1', 'comp_index': 0, 'label': 'Comp 1', 'position': [10, 20], + 'overexposure_rejected_count': 0, + }, + { + 'key': 'comp2', 'comp_index': 1, 'label': 'Comp 2', 'position': [30, 40], + 'overexposure_rejected_count': 0, + }, + ], + { + '000-BPW-929': { + 'pos': [10, 20], + 'mag': 11.85, + 'error': 0.046, + 'mag_band': 'V', + 'catalog_source': 'AAVSO VSP', + 'is_aavso_vsp': True, + 'catalog_ra': 18.0, + 'catalog_dec': 35.0, + }, + '000-BMX-191': { + 'pos': [30, 40], + 'mag': 12.121, + 'error': 0.005, + 'mag_band': 'V', + 'catalog_source': 'AAVSO VSP', + 'is_aavso_vsp': True, + 'catalog_ra': 18.1, + 'catalog_dec': 35.1, + }, + }, + { + 'comp1': np.full(frame_count, 2000.0), + 'comp2': np.full(frame_count, 1000.0), + }, + observed_filter='Clear', + min_members=1, + max_members=1, + ) + + assert [member['key'] for member in selection['members']] == ['comp1'] + assert selection['members'][0]['label'] == '000-BPW-929' + assert selection['members'][0]['star']['catalog_source'] == 'AAVSO VSP' + + +def test_stellar_variability_rejects_comparison_that_steps_across_acquisition_gap(): + frame_count = 180 + cadence_days = 6.0 / 86400.0 + times = 2460000.0 + (np.arange(frame_count, dtype=float) * cadence_days) + times[90:] += 120.0 / 86400.0 + ranked_summaries = [ + { + 'key': f'comp{index}', + 'comp_index': index - 1, + 'label': f'Comp {index}', + 'position': [10 * index, 20 * index], + 'overexposure_rejected_count': 0, + } + for index in range(1, 5) + ] + calibration_stars = { + f'C{index}': { + 'pos': [10 * index, 20 * index], + 'mag': 12.0 + (0.1 * index), + 'error': 0.01, + 'mag_band': 'V', + 'catalog_source': 'Synthetic catalog', + } + for index in range(1, 5) + } + phase = np.linspace(0, 4 * np.pi, frame_count) + common = 1000.0 * (1.0 + (0.01 * np.sin(phase))) + comp_flux_map = { + f'comp{index}': common * (1.0 + (0.001 * index * np.cos(phase))) + for index in range(1, 5) + } + # A +0.06 mag instrumental discontinuity in comparison 1. + comp_flux_map['comp1'] = comp_flux_map['comp1'].copy() + comp_flux_map['comp1'][90:] *= 10.0 ** (-0.4 * 0.06) + + selection = exotic_module.select_stellar_variability_ensemble_members( + ranked_summaries, + calibration_stars, + comp_flux_map, + observed_filter='V', + min_members=1, + max_members=1, + times=times, + ) + + assert selection['gap_stability']['applied'] is True + assert selection['gap_stability']['boundaries'][0]['source_index'] == 90 + assert selection['gap_stability']['candidates']['comp1']['rejected'] is True + assert selection['gap_stability']['candidates']['comp1'][ + 'maximum_absolute_step_magnitude' + ] == pytest.approx(0.06, abs=0.003) + assert selection['gap_stability']['candidates']['comp1']['label'] == 'C1' + assert selection['gap_stability']['candidates']['comp1']['position'] == [10, 20] + assert selection['gap_stability']['candidates']['comp1']['catalog_magnitude_band'] == 'V' + assert selection['gap_stability']['candidates']['comp1']['catalog_source'] == 'Synthetic catalog' + assert selection['members'][0]['key'] != 'comp1' + rejected = {item['key']: item for item in selection['rejected']} + assert rejected['comp1']['label'] == 'C1' + assert rejected['comp1']['position'] == [10, 20] + assert 'changed discontinuously' in rejected['comp1']['reason'] + + +def test_fortuitous_output_error_limit_relaxes_only_for_flagged_bv_catalog_reference(): + assert exotic_module.fortuitous_output_magnitude_error_limit([ + {'star': {'mag_band': 'V', 'uses_relaxed_bv_error_limit': True}} + ]) == pytest.approx(0.10) + assert exotic_module.fortuitous_output_magnitude_error_limit([ + {'star': {'mag_band': 'B', 'uses_relaxed_bv_error_limit': True}} + ]) == pytest.approx(0.10) + assert exotic_module.fortuitous_output_magnitude_error_limit([ + {'star': {'mag_band': 'g', 'uses_relaxed_bv_error_limit': True}} + ]) == pytest.approx(0.05) + assert exotic_module.fortuitous_output_magnitude_error_limit([ + {'star': {'mag_band': 'V', 'uses_relaxed_bv_error_limit': False}} + ]) == pytest.approx(0.05) + + metadata = exotic_module.fortuitous_variable_target_metadata({ + 'name': 'Synthetic', + 'reference_mode': 'single_comparison', + 'output_magnitude_error_limit': 0.10, + }) + assert metadata['detection_magnitude_error_limit'] == pytest.approx(0.05) + assert metadata['output_magnitude_error_limit'] == pytest.approx(0.10) + + +def test_stellar_variability_ensemble_error_clip_does_not_reject_below_point_zero_one_mag(): + candidates = [ + {'magnitude_error': error} + for error in (0.0010, 0.0011, 0.0012, 0.0090, 0.0110) + ] + + keep, summary = exotic_module.stellar_variability_ensemble_calibration_error_clip(candidates) + + assert keep.tolist() == [True, True, True, True, False] + assert summary['high_threshold'] == pytest.approx(0.01) + assert summary['minimum_high_threshold'] == pytest.approx(0.01) + + +def test_automatic_comparison_merge_deduplicates_only_added_sources(): + merged, messages = exotic_module.merge_automatic_comparison_star_coords( + [[10.0, 20.0], [11.0, 20.0]], + [[10.4, 20.3], [50.0, 60.0], [50.5, 60.2]], + duplicate_radius_pixels=2.0, + ) + + # Nearby primary/user selections remain intentional; automatic additions + # cannot repeat either a primary source or an earlier automatic source. + assert merged == [[10.0, 20.0], [11.0, 20.0], [50.0, 60.0]] + assert len(messages) == 2 + + +def test_full_field_vsx_variables_are_removed_from_science_comparisons(): + comparison_stars = [ + [4969.0, 1695.0], + [5221.0, 2714.0], + [3923.0, 1362.0], + ] + fortuitous_variables = [ + {'name': 'DI Her', 'pos': [4971.55, 1695.42]}, + { + 'name': 'ASASSN-V J185327.35+241158.6', + 'x': 3926.38, + 'y': 1360.36, + }, + ] + + retained, rejected = exotic_module.filter_comparison_stars_against_fortuitous_variables( + comparison_stars, + fortuitous_variables, + duplicate_radius_pixels=10.0, + ) + + assert retained == [[5221.0, 2714.0]] + assert [item['comparison_index'] for item in rejected] == [0, 2] + assert [item['variable_name'] for item in rejected] == [ + 'DI Her', + 'ASASSN-V J185327.35+241158.6', + ] + assert rejected[0]['distance_pixels'] == pytest.approx(2.5840, abs=1.0e-3) + assert rejected[1]['distance_pixels'] == pytest.approx(3.7563, abs=1.0e-3) + + +def test_tracked_vsx_overexposure_warning_does_not_call_variable_a_comparison_star(): + messages = [] + status = PlateStatus(lambda message, **kwargs: messages.append(message)) + status.setCurrentFilename('frame.fits') + + status.overexposedWarning( + 12, + 4973.2, + 1676.6, + 58981.5, + starLabel='Tracked VSX variable DI Her', + ) + + assert messages[0] == ( + 'Tracked VSX variable DI Her is overexposed in file frame.fits; ' + 'aperture pixels near [4973.2, 1676.6] exceeded 58981.5.' + ) + assert 'repeated frame-level star warnings are aggregated' in messages[1] + assert 'Comparison star' not in messages[0] + + +def test_stellar_variability_ensemble_caps_at_five_by_target_color_and_magnitude(): + frame_count = 8 + target_match = { + 'mag': 12.0, + 'error': 0.01, + 'mag_band': 'V', + 'catalog_row': {'Bmag': 12.5, 'Vmag': 12.0}, + } + candidate_values = [ + (10.0, -0.5), + (11.0, 0.0), + (12.1, 0.55), + (12.2, 0.60), + (11.9, 0.45), + (12.3, 0.40), + (12.0, 0.52), + ] + ranked_summaries = [] + calibration_stars = {} + comp_flux_map = {} + for index, (magnitude, color) in enumerate(candidate_values, start=1): + key = f'comp{index}' + position = [index * 10, index * 10 + 1] + ranked_summaries.append({ + 'key': key, + 'comp_index': index - 1, + 'label': f'Comp {index}', + 'position': position, + 'overexposure_rejected_count': 0, + }) + calibration_stars[f'C{index}'] = { + 'pos': position, + 'mag': magnitude, + 'error': 0.01, + 'mag_band': 'V', + 'catalog_row': {'Bmag': magnitude + color, 'Vmag': magnitude}, + } + comp_flux_map[key] = np.full(frame_count, 10000.0 - index * 100.0) + + selection = exotic_module.select_stellar_variability_ensemble_members( + ranked_summaries, + calibration_stars, + comp_flux_map, + observed_filter='V', + target_catalog_match=target_match, + ) + + assert selection['prelimit_member_count'] == 7 + assert selection['member_limit'] == 5 + assert [member['key'] for member in selection['members']] == [ + 'comp7', 'comp3', 'comp5', 'comp4', 'comp6', + ] + assert all(member['color_delta'] is not None for member in selection['members']) + assert all(member['magnitude_delta'] is not None for member in selection['members']) + limited_keys = { + rejected['key'] + for rejected in selection['rejected'] + if 'closest to the target' in rejected['reason'] + } + assert limited_keys == {'comp1', 'comp2'} + + expanded_selection = exotic_module.select_stellar_variability_ensemble_members( + ranked_summaries, + calibration_stars, + comp_flux_map, + observed_filter='V', + target_catalog_match=target_match, + max_members=7, + ) + + assert expanded_selection['member_limit'] == 7 + assert len(expanded_selection['members']) == 7 + assert not any( + 'closest to the target' in rejected['reason'] + for rejected in expanded_selection['rejected'] + ) + + +def test_stellar_variability_ensemble_uses_gaia_bp_rp_when_local_colors_are_missing( + monkeypatch): + calls = [] + + def fake_gaia_lookup(ra, dec, max_separation_arcsec): + calls.append((ra, dec, max_separation_arcsec)) + return { + 'color': 1.2 if ra == 10.0 else 1.25, + 'label': 'BP-RP', + } + + monkeypatch.setattr(exotic_module, 'nextastro_gaia_bp_rp_for_coordinate', fake_gaia_lookup) + selection = exotic_module.select_stellar_variability_ensemble_members( + [{ + 'key': 'comp1', + 'comp_index': 0, + 'label': 'Comp 1', + 'position': [10, 20], + 'overexposure_rejected_count': 0, + }], + { + 'NextAstro-1': { + 'pos': [10, 20], + 'mag': 12.1, + 'error': 0.01, + 'mag_band': 'V', + 'catalog_row': {'ra': 11.0, 'dec': 20.0, 'Vmag': 12.1}, + }, + }, + {'comp1': np.full(8, 1000.0)}, + observed_filter='V', + target_catalog_match={ + 'mag': 12.0, + 'error': 0.01, + 'mag_band': 'V', + 'catalog_row': {'ra': 10.0, 'dec': 20.0, 'Vmag': 12.0}, + }, + min_members=1, + max_members=1, + ) + + assert selection['target_catalog_profile']['color_label'] == 'BP-RP' + assert selection['target_catalog_profile']['color'] == pytest.approx(1.2) + assert selection['members'][0]['color_label'] == 'BP-RP' + assert selection['members'][0]['color_delta'] == pytest.approx(0.05) + assert calls == [(10.0, 20.0, 2.0), (11.0, 20.0, 2.0)] + + +def test_stellar_variability_ensemble_rejects_duplicate_catalog_sources(): + frame_count = 8 + ranked_summaries = [ + { + 'key': 'comp1', 'comp_index': 0, 'label': 'Comp 1', 'position': [10, 20], + 'overexposure_rejected_count': 0, + }, + { + 'key': 'comp2', 'comp_index': 1, 'label': 'Comp 2', 'position': [30, 40], + 'overexposure_rejected_count': 0, + }, + { + 'key': 'comp3', 'comp_index': 2, 'label': 'Comp 3', 'position': [50, 60], + 'overexposure_rejected_count': 0, + }, + ] + calibration_stars = { + 'NextAstro-111': { + 'pos': [10, 20], 'mag': 12.0, 'error': 0.01, 'mag_band': 'V', 'source_id': 111, + }, + 'NextAstro-111-2': { + 'pos': [30, 40], 'mag': 12.0, 'error': 0.01, 'mag_band': 'V', 'source_id': 111, + }, + 'NextAstro-222': { + 'pos': [50, 60], 'mag': 12.5, 'error': 0.01, 'mag_band': 'V', 'source_id': 222, + }, + } + comp_flux_map = { + 'comp1': np.full(frame_count, 3000.0), + 'comp2': np.full(frame_count, 2000.0), + 'comp3': np.full(frame_count, 1000.0), + } + + selection = exotic_module.select_stellar_variability_ensemble_members( + ranked_summaries, + calibration_stars, + comp_flux_map, + observed_filter='V', + ) + + assert [member['key'] for member in selection['members']] == ['comp1', 'comp3'] + duplicate = next(item for item in selection['rejected'] if item['key'] == 'comp2') + assert 'duplicate catalog source' in duplicate['reason'] + + +def test_discover_fortuitous_vsx_variables_filters_on_count_rate_error_and_classifies(monkeypatch): + class FakeWcs: + def pixel_to_world_values(self, x_value, y_value): + return x_value, y_value + + def world_to_pixel_values(self, ra_value, dec_value): + return ra_value, dec_value + + reference_image = np.zeros((80, 80), dtype=float) + reference_image[20, 20] = 10000.0 + reference_image[40, 40] = 100.0 + monkeypatch.setattr(exotic_module, 'search_wcs', lambda _path: FakeWcs()) + monkeypatch.setattr( + exotic_module, + 'vsx_field_query', + lambda *args, **kwargs: [ + { + 'Name': 'Bright VSX', 'AUID': '000-AAA-001', + 'RA2000': 20.0, 'Declination2000': 20.0, + 'Period': '5.0', 'MaxMag': '12.0 V', 'MinMag': '12.5 V', + }, + { + 'Name': 'Faint VSX', 'AUID': '000-AAA-002', + 'RA2000': 40.0, 'Declination2000': 40.0, + 'Period': '20.0', 'MaxMag': '15.0 V', 'MinMag': '15.2 V', + }, + ], + ) + + variables = exotic_module.discover_fortuitous_vsx_variables( + 'synthetic.wcs', + reference_image.shape, + 1.0, + reference_image, + 'V', + target_pixel=[60, 60], + exposure_seconds=60.0, + ) + + assert len(variables) == 1 + assert variables[0]['name'] == 'Bright VSX' + assert variables[0]['estimated_magnitude_error'] < 0.05 + assert variables[0]['category'] == 'optimal_variables' + assert variables[0]['period_days'] == pytest.approx(5.0) + assert variables[0]['amplitude_mag'] == pytest.approx(0.5) + assert exotic_module.fortuitous_variable_category(20.0, 0.2) == 'normal' + + +def test_fortuitous_reference_error_estimate_includes_sky_noise(monkeypatch): + reference_image = np.full((80, 80), 1000.0, dtype=float) + reference_image[40, 40] += 5000.0 + monkeypatch.setattr( + exotic_module, + 'skybg_phot', + lambda *args, **kwargs: (1000.0, 100.0, 500.0), + ) + + estimate = exotic_module.estimated_magnitude_error_from_reference_count_rate( + reference_image, + 40.0, + 40.0, + exposure_seconds=60.0, + gain_e_per_adu=1.0, + ) + + source_only_error = ( + (2.5 / np.log(10.0)) + * exotic_module.source_flux_uncertainty_from_counts(5000.0, gain_e_per_adu=1.0) + / 5000.0 + ) + assert source_only_error < 0.05 + assert estimate['estimated_magnitude_error'] > 0.05 + assert estimate['reference_noise_components_adu']['sky_aperture'] > 0 + assert estimate['reference_noise_components_adu']['sky_estimate'] > 0 + + +def test_calibrated_stellar_variability_ensemble_combines_catalog_zero_points(): + frame_count = 7 + target_flux = np.full(frame_count, 1000.0) + target_error = np.full(frame_count, 1.0) + comp_flux_map = { + 'comp1': np.full(frame_count, 500.0), + 'comp2': np.full(frame_count, 250.0), + } + comp_error_map = { + 'comp1': np.full(frame_count, 1.0), + 'comp2': np.full(frame_count, 1.0), + } + members = [ + { + 'key': 'comp1', + 'magnitude': 12.0, + 'magnitude_error': 0.01, + 'summary': {'ensemble_frame_keep_mask': np.ones(frame_count, dtype=bool)}, + }, + { + 'key': 'comp2', + 'magnitude': 12.0 + 2.5 * np.log10(2.0), + 'magnitude_error': 0.01, + 'summary': {'ensemble_frame_keep_mask': np.ones(frame_count, dtype=bool)}, + }, + ] + + result = exotic_module.build_stellar_variability_calibrated_ensemble_series( + target_flux, + target_error, + comp_flux_map, + comp_error_map, + members, + ) + + expected_target_magnitude = 12.0 - 2.5 * np.log10(2.0) + assert result['applied'] is True + np.testing.assert_allclose(result['magnitude'], expected_target_magnitude, atol=1.0e-10) + np.testing.assert_allclose(result['relative_flux'], 1.0, atol=1.0e-10) + np.testing.assert_allclose(result['raw_reference_flux'], 375.0, atol=1.0e-10) + assert np.all(np.isfinite(result['raw_reference_flux_error'])) + np.testing.assert_array_equal(result['valid_member_count'], np.full(frame_count, 2)) + assert np.all(result['magnitude_error'] > 0) + + +def test_calibrated_stellar_variability_ensemble_rejects_frame_missing_any_member(): + frame_count = 7 + target_flux = np.full(frame_count, 1000.0) + target_error = np.full(frame_count, 1.0) + comp_flux_map = { + 'comp1': np.full(frame_count, 500.0), + 'comp2': np.full(frame_count, 250.0), + 'comp3': np.full(frame_count, 400.0), + } + comp_error_map = { + key: np.full(frame_count, 1.0) + for key in comp_flux_map + } + members = [ + { + 'key': 'comp1', + 'magnitude': 12.0, + 'magnitude_error': 0.01, + 'summary': {'ensemble_frame_keep_mask': np.ones(frame_count, dtype=bool)}, + }, + { + 'key': 'comp2', + 'magnitude': 12.0 + 2.5 * np.log10(2.0), + 'magnitude_error': 0.01, + 'summary': {'ensemble_frame_keep_mask': np.ones(frame_count, dtype=bool)}, + }, + { + 'key': 'comp3', + 'magnitude': 12.0 + 2.5 * np.log10(1.25), + 'magnitude_error': 0.01, + 'summary': { + 'ensemble_frame_keep_mask': np.array( + [True, True, True, False, True, True, True], + dtype=bool, + ), + }, + }, + ] + + result = exotic_module.build_stellar_variability_calibrated_ensemble_series( + target_flux, + target_error, + comp_flux_map, + comp_error_map, + members, + minimum_members=2, + ) + + assert result['applied'] is True + assert result['valid_member_count'][3] == 2 + assert np.isnan(result['magnitude'][3]) + assert np.isnan(result['relative_flux'][3]) + finite_indices = np.flatnonzero(np.isfinite(result['magnitude'])) + np.testing.assert_array_equal(finite_indices, [0, 1, 2, 4, 5, 6]) + + +def test_build_stellar_variability_ensemble_params_preserves_member_metadata(monkeypatch, tmp_path): + captured = {} + monkeypatch.setattr( + exotic_module, + 'plot_stellar_variability', + lambda params, save, target, label: captured.update( + params=params, + save=save, + target=target, + label=label, + ), + ) + fit = types.SimpleNamespace( + time=np.array([2460000.105, 2460000.205]), + jd_times=np.array([2460000.1, 2460000.2]), + data=np.array([1.0, 1.1]), + dataerr=np.array([0.01, 0.011]), + airmass=np.array([1.1, 1.2]), + airmass_model=np.ones(2), + transit=np.ones(2), + stellar_variability_only=True, + stellar_variability_target_flux=np.array([500.0, 550.0]), + stellar_variability_comp_flux=np.full(2, 1000.0), + stellar_variability_target_flux_error=np.full(2, 2.0), + stellar_variability_comp_flux_error=np.full(2, 3.0), + stellar_variability_ensemble_magnitudes=np.array([12.30, 12.31]), + stellar_variability_ensemble_magnitude_errors=np.array([0.01, 0.011]), + stellar_variability_ensemble_members=[ + { + 'label': 'C1', 'position': [10, 20], 'magnitude': 12.0, + 'magnitude_error': 0.01, + 'star': {'catalog_source': 'Catalog A', 'ra': 10.1, 'dec': -20.1}, + }, + { + 'label': 'C2', 'position': [30, 40], 'magnitude': 12.5, + 'magnitude_error': 0.011, + 'star': {'catalog_source': 'Catalog B', 'ra': 10.2, 'dec': -20.2}, + }, + ], + ) + + params = exotic_module.build_stellar_variability_ensemble_params_from_fit( + fit, + tmp_path, + 'Target Star', + observed_filter='MObs CV', + ) + + assert len(params) == 2 + assert [row['time'] for row in params] == pytest.approx([2460000.105, 2460000.205]) + assert params[0]['cname'] == 'ENSEMBLE (2 stars)' + assert params[0]['cmag'] is None + assert params[0]['mag_band'] == 'ClearV' + assert params[0]['catalog_mag_band'] == 'V' + assert params[0]['differential_mag'] == pytest.approx(-2.5 * np.log10(0.5)) + assert params[1]['differential_mag'] == pytest.approx(-2.5 * np.log10(0.55)) + assert params[0]['differential_mag_err'] > 0 + assert params[0]['ensemble_member_labels'] == ['C1', 'C2'] + assert params[0]['ensemble_member_catalog_errors'] == [0.01, 0.011] + assert params[0]['ensemble_member_ra_degs'] == [10.1, 10.2] + assert params[0]['ensemble_member_dec_degs'] == [-20.1, -20.2] + assert params[0]['ensemble_members'][0]['ra_deg'] == pytest.approx(10.1) + assert params[0]['ensemble_members'][1]['dec_deg'] == pytest.approx(-20.2) + assert fit.stellar_variability_params == params + assert captured['label'] == 'ENSEMBLE (2 stars)' + + r_params = exotic_module.build_stellar_variability_ensemble_params_from_fit( + fit, + tmp_path, + 'Target Star', + observed_filter='R', + ) + + assert r_params[0]['mag_band'] == 'rp' + assert r_params[0]['catalog_mag_band'] == 'r' + + +def test_stellar_variability_ensemble_selection_json_lists_color_and_magnitude(monkeypatch, tmp_path): + monkeypatch.setattr(exotic_module, 'plot_stellar_variability', lambda *args, **kwargs: None) + member = { + 'selection_rank': 1, + 'key': 'comp1', + 'label': 'C1', + 'position': [10, 20], + 'magnitude': 12.1, + 'magnitude_error': 0.01, + 'color': 0.55, + 'color_label': 'B-V', + 'target_color': 0.50, + 'target_color_label': 'B-V', + 'color_delta': 0.05, + 'target_magnitude': 12.0, + 'magnitude_delta': 0.1, + 'color_magnitude_similarity_score': np.hypot(0.05, 0.1), + 'median_flux': 5000.0, + 'star': { + 'ra': 10.1, + 'dec': -20.2, + 'mag_band': 'V', + 'catalog_source': 'Synthetic catalog', + }, + } + fit = types.SimpleNamespace( + jd_times=np.array([2460000.1]), + airmass=np.array([1.1]), + stellar_variability_ensemble_magnitudes=np.array([12.3]), + stellar_variability_ensemble_magnitude_errors=np.array([0.02]), + stellar_variability_ensemble_members=[member], + stellar_variability_ensemble_selection={ + 'members': [member], + 'rejected': [{'key': 'comp2', 'reason': 'not among the 5 closest'}], + 'member_limit': 5, + 'prelimit_member_count': 6, + 'calibration_error_clip': {'high_threshold': 0.03}, + 'target_catalog_profile': { + 'magnitude': 12.0, + 'magnitude_band': 'V', + 'color': 0.5, + 'color_label': 'B-V', + }, + }, + ) + + exotic_module.build_stellar_variability_ensemble_params_from_fit( + fit, + tmp_path, + 'Target Star', + observed_filter='V', + observation_date='2024-01-02', + ) + + output_path = next(tmp_path.glob('EnsembleSelection_TargetStar_2024-01-02.json')) + payload = json.loads(output_path.read_text(encoding='utf-8')) + assert payload['ensemble']['maximum_members'] == 5 + assert payload['ensemble']['member_count_before_five_star_limit'] == 6 + assert payload['ensemble']['per_frame_required_members'] == 1 + assert 'every selected ensemble member is valid' in payload['ensemble']['selection_rule'] + assert payload['target']['catalog_profile']['color'] == pytest.approx(0.5) + assert payload['ensemble']['members'][0]['color_delta'] == pytest.approx(0.05) + assert payload['ensemble']['members'][0]['magnitude_delta'] == pytest.approx(0.1) + + +def test_process_fortuitous_variables_write_independent_and_combined_aid_products(monkeypatch, tmp_path): + monkeypatch.setattr(exotic_module, 'plot_stellar_variability', lambda *args, **kwargs: None) + monkeypatch.setattr( + exotic_module, + 'psf_quality_mask_for_key', + lambda psf_data, key, frame_count, psf_flux_data=None: np.ones(frame_count, dtype=bool), + ) + frame_count = 12 + times = np.linspace(2460000.0, 2460000.1, frame_count) + quality_mask = np.ones(frame_count, dtype=bool) + comparison_calibration = { + 'method': 'aperture', + 'method_label': 'Aperture photometry', + 'a': 0, + 'an': 0, + 'field_image_keep_mask': quality_mask, + 'comp_summaries': [ + { + 'key': 'comp1', 'comp_index': 0, 'label': 'Comp 1', 'position': [10, 20], + 'aggregate_score': 0.001, 'coverage_rejected': False, + 'suitability_outlier_rejected': False, 'overexposure_rejected_count': 0, + 'psf_quality_keep_mask': quality_mask, + 'ensemble_frame_keep_mask': quality_mask, + }, + { + 'key': 'comp2', 'comp_index': 1, 'label': 'Comp 2', 'position': [30, 40], + 'aggregate_score': 0.002, 'coverage_rejected': False, + 'suitability_outlier_rejected': False, 'overexposure_rejected_count': 0, + 'psf_quality_keep_mask': quality_mask, + 'ensemble_frame_keep_mask': quality_mask, + }, + ], + } + psf_data = { + # The exoplanet target is unusable in every frame. Fortuitous-variable + # processing must remain independent of that target-specific mask. + 'target': np.full((frame_count, 7), np.nan), + 'comp1': np.ones((frame_count, 7)), + 'comp2': np.ones((frame_count, 7)), + 'comp3': np.ones((frame_count, 7)), + 'comp4': np.ones((frame_count, 7)), + } + aper_data = { + 'target': np.full((frame_count, 1, 1), 1000.0), + 'comp1': np.full((frame_count, 1, 1), 500.0), + 'comp1_unc': np.full((frame_count, 1, 1), 1.0), + 'comp2': np.full((frame_count, 1, 1), 250.0), + 'comp2_unc': np.full((frame_count, 1, 1), 1.0), + 'comp3': ( + 800.0 * (1.0 + 0.02 * np.sin(np.linspace(0, 2 * np.pi, frame_count))) + )[:, None, None], + 'comp3_unc': np.full((frame_count, 1, 1), 1.0), + 'comp4': ( + 700.0 * (1.0 + 0.01 * np.cos(np.linspace(0, 2 * np.pi, frame_count))) + )[:, None, None], + 'comp4_unc': np.full((frame_count, 1, 1), 1.0), + } + # One otherwise valid frame has an internal target error above 0.05 mag. + aper_data['comp3_unc'][2, 0, 0] = 80.0 + calibrations = { + 'C1': { + 'pos': [10, 20], 'mag': 12.0, 'error': 0.01, 'mag_band': 'V', + 'ra': 10.1, 'dec': -20.1, + 'catalog_source': 'Synthetic catalog', + 'catalog_row': {'Bmag': 12.5, 'Vmag': 12.0}, + }, + 'C2': { + 'pos': [30, 40], 'mag': 12.75, 'error': 0.011, 'mag_band': 'V', + 'ra': 10.2, 'dec': -20.2, + 'catalog_source': 'Synthetic catalog', + 'catalog_row': {'Bmag': 13.35, 'Vmag': 12.75}, + }, + } + variable = { + 'name': 'Synthetic VSX', + 'auid': '000-AAA-001', + 'variable_type': 'EA', + 'period_days': 5.0, + 'amplitude_mag': 0.5, + 'category': 'optimal_variables', + 'ra': 10.0, + 'dec': -20.0, + 'pos': [50, 60], + 'tracking_key': 'comp3', + 'aperture_flux_adu': 800.0, + 'count_rate_adu_per_second': 13.3, + 'estimated_magnitude_error': 0.04, + 'catalog_match': { + 'mag': 12.4, + 'error': 0.02, + 'mag_band': 'V', + 'catalog_row': {'Bmag': 12.95, 'Vmag': 12.4}, + }, + } + second_variable = { + **variable, + 'name': 'Synthetic VSX 2', + 'auid': '000-AAA-002', + 'pos': [70, 80], + 'tracking_key': 'comp4', + 'aperture_flux_adu': 700.0, + 'count_rate_adu_per_second': 11.7, + } + info_dict = { + 'save': str(tmp_path), + 'date': '2024-01-02', + 'aavso_num': 'RTZ', + 'camera': 'CCD', + 'filter': 'V', + 'lat': '+32.4', + 'long': '-110.7', + 'elev': 2600, + } + + variable_overexposed = np.zeros(frame_count, dtype=bool) + variable_overexposed[:2] = True + results = exotic_module.process_fortuitous_variables( + [variable, second_variable], + comparison_calibration, + calibrations, + times, + times, + np.linspace(1.1, 1.3, frame_count), + psf_data, + aper_data, + info_dict, + comp_overexposed_masks={'comp3': variable_overexposed}, + exposure_times_seconds=np.full(frame_count, 60.0), + observed_filter='V', + use_single_comparison=False, + maximum_number_of_ensemble_comparisons_for_stellar_variability=2, + ) + + assert results[0]['status'] == 'completed' + assert results[0]['input_frame_count'] == frame_count + assert results[0]['target_overexposure_rejected_frame_count'] == 2 + assert results[0]['output_magnitude_error_rejected_frame_count'] == 1 + assert results[0]['point_count'] == frame_count - 3 + variable_dir = tmp_path / 'variables' / 'optimal_variables' / 'SyntheticVSX' + assert next( + (variable_dir / 'AAVSO_Files').glob('AID_AAVSO_SyntheticVSX_2024-01-02.txt') + ).is_file() + second_variable_dir = ( + tmp_path / 'variables' / 'optimal_variables' / 'SyntheticVSX2' + ) + assert next( + (second_variable_dir / 'AAVSO_Files').glob('AID_AAVSO_SyntheticVSX2_2024-01-02.txt') + ).is_file() + assert next(variable_dir.glob('EnsembleSelection_SyntheticVSX_2024-01-02.json')).is_file() + assert next(variable_dir.glob('StellarVariability_SyntheticVSX_2024-01-02.csv')).is_file() + combined_aid_path = ( + tmp_path / 'variables' / 'AAVSO_Files' / 'AID_AAVSO_FortuitousVariables_2024-01-02.txt' + ) + combined_aid_text = combined_aid_path.read_text(encoding='utf-8') + combined_aid_rows = [ + line for line in combined_aid_text.splitlines() + if line and not line.startswith('#') + ] + assert combined_aid_text.count('#TYPE=EXTENDED') == 1 + assert '#ENSEMBLE-COMPARISONS-XC=' not in combined_aid_text + assert len(combined_aid_rows) == sum(result['point_count'] for result in results) + assert {row.split(',', 1)[0] for row in combined_aid_rows} == { + '000-AAA-001', + '000-AAA-002', + } + manifest = json.loads( + next((tmp_path / 'variables').glob('FortuitousVariables_2024-01-02.json')).read_text( + encoding='utf-8' + ) + ) + assert manifest['combined_aid'] == str(combined_aid_path) + assert manifest['variables'][0]['ensemble_member_count'] == 2 + assert manifest['variables'][0]['comparison_gap_stability']['applied'] is False + assert manifest['variables'][0]['comparison_gap_rejected_candidates'] == [] + assert manifest['variables'][0]['output_magnitude_error_rejected_frame_count'] == 1 + assert manifest['variables'][0]['output_magnitude_error_max'] > 0.05 + selection = json.loads( + next(variable_dir.glob('EnsembleSelection_SyntheticVSX_2024-01-02.json')).read_text( + encoding='utf-8' + ) + ) + assert selection['target']['input_frame_count'] == frame_count + assert selection['target']['target_overexposure_rejected_frame_count'] == 2 + assert selection['target']['output_magnitude_error_rejected_frame_count'] == 1 + assert selection['target']['valid_output_frame_count'] == frame_count - 3 + assert 'exoplanet target overexposure mask is not applied' in ( + selection['target']['saturation_rejection_scope'] + ) + aid_text = next( + (variable_dir / 'AAVSO_Files').glob('AID_AAVSO_SyntheticVSX_2024-01-02.txt') + ).read_text(encoding='utf-8') + assert ( + '#NAME,DATE,MAG,MERR,FILT,TRANS,MTYPE,CNAME,CMAG,KNAME,KMAG,AMASS,' + 'GROUP,CHART,NOTES,DIFFMAG,DIFFERR' + ) in aid_text + assert '|DIFFMAG=' not in aid_text + assert '|DIFFERR=' not in aid_text + aid_data_row = next(line for line in aid_text.splitlines() if not line.startswith('#')) + assert aid_data_row.split(',')[-3] == 'na' + assert aid_data_row.split(',')[-2] != 'na' + assert aid_data_row.split(',')[-1] != 'na' + ensemble_header = next( + line for line in aid_text.splitlines() + if line.startswith('#ENSEMBLE-COMPARISONS-XC=') + ) + ensemble_metadata = json.loads(ensemble_header.split('=', 1)[1]) + assert ensemble_metadata['member_count'] == 2 + assert ensemble_metadata['members'][0]['ra_deg'] == pytest.approx(10.1) + assert ensemble_metadata['members'][0]['dec_deg'] == pytest.approx(-20.1) + assert ensemble_metadata['members'][1]['ra_deg'] == pytest.approx(10.2) + assert ensemble_metadata['members'][1]['dec_deg'] == pytest.approx(-20.2) + csv_path = next(variable_dir.glob('StellarVariability_SyntheticVSX_2024-01-02.csv')) + csv_text = csv_path.read_text(encoding='utf-8') + assert 'Apparent Magnitude' in csv_text.splitlines()[0] + assert 'Raw Differential Magnitude' in csv_text.splitlines()[0] + exported_errors = [ + float(row.split(',')[3]) + for row in csv_text.splitlines()[1:] + if row.strip() + ] + assert exported_errors + assert max(exported_errors) < 0.05 + + single_root = tmp_path / 'single' + single_info = {**info_dict, 'save': str(single_root)} + single_results = exotic_module.process_fortuitous_variables( + [variable], + comparison_calibration, + calibrations, + times, + times - 0.005, + np.linspace(1.1, 1.3, frame_count), + psf_data, + aper_data, + single_info, + comp_overexposed_masks={'comp3': variable_overexposed}, + exposure_times_seconds=np.full(frame_count, 60.0), + observed_filter='V', + ) + + assert single_results[0]['status'] == 'completed' + assert single_results[0]['reference_mode'] == 'single_comparison' + assert single_results[0]['comparison_member_count'] == 1 + assert single_results[0]['comparison_label'] == 'C2' + single_dir = ( + single_root / 'variables' / 'optimal_variables' / 'SyntheticVSX' + ) + assert not list(single_dir.glob('EnsembleSelection_*.json')) + single_csv = next(single_dir.glob('StellarVariability_SyntheticVSX_2024-01-02.csv')) + single_csv_rows = [ + line for line in single_csv.read_text(encoding='utf-8').splitlines()[1:] + if line.strip() + ] + assert float(single_csv_rows[0].split(',')[0]) == pytest.approx(times[3]) + single_aid = next( + (single_dir / 'AAVSO_Files').glob('AID_AAVSO_SyntheticVSX_2024-01-02.txt') + ) + single_aid_text = single_aid.read_text(encoding='utf-8') + assert '#ENSEMBLE-COMPARISONS-XC=' not in single_aid_text + single_aid_row = next( + line for line in single_aid_text.splitlines() + if line and not line.startswith('#') + ) + assert single_aid_row.split(',')[7] == 'C2' + + failed_root = tmp_path / 'failed' + failed_results = exotic_module.process_fortuitous_variables( + [variable], + comparison_calibration, + {}, + times, + times, + np.linspace(1.1, 1.3, frame_count), + psf_data, + aper_data, + {**info_dict, 'save': str(failed_root)}, + comp_overexposed_masks={'comp3': variable_overexposed}, + exposure_times_seconds=np.full(frame_count, 60.0), + observed_filter='V', + ) + + assert failed_results[0]['status'] == 'completed' + assert failed_results[0]['apparent_magnitude_point_count'] == 0 + assert failed_results[0]['apparent_magnitude_error'] + differential_only_dir = ( + failed_root / 'variables' / 'optimal_variables' / 'SyntheticVSX' + ) + assert differential_only_dir.exists() + assert next(differential_only_dir.glob('DifferentialMagnitude_*.csv')).is_file() + assert not list(differential_only_dir.glob('StellarVariability_*.csv')) + assert not list((differential_only_dir / 'AAVSO_Files').glob('AID_AAVSO_*.txt')) + failed_manifest = json.loads( + next((failed_root / 'variables').glob('FortuitousVariables_2024-01-02.json')).read_text( + encoding='utf-8' + ) + ) + assert failed_manifest['variables'][0]['status'] == 'completed' + assert failed_manifest['variables'][0]['apparent_magnitude_point_count'] == 0 + assert failed_manifest['variables'][0]['differential_magnitude_csv'] + + +def test_stellar_variability_selector_uses_calibrated_ensemble_by_default(monkeypatch, tmp_path): + monkeypatch.setattr(exotic_module, 'plot_stellar_variability', lambda *args, **kwargs: None) + logged = [] + monkeypatch.setattr(exotic_module, 'log_info', lambda message, **kwargs: logged.append(message)) + frame_count = 12 + times = np.linspace(10.2, 10.3, frame_count) + target_flux = 1000.0 * (1.0 + np.linspace(-0.002, 0.002, frame_count)) + comp1_flux = np.full(frame_count, 500.0) + comp2_flux = np.full(frame_count, 250.0) + quality_mask = np.ones(frame_count, dtype=bool) + comp_summaries = [ + { + 'key': 'comp1', 'comp_index': 0, 'label': 'Comp 1', 'position': [10, 20], + 'aggregate_score': 0.001, 'coverage_rejected': False, + 'suitability_outlier_rejected': False, 'overexposure_rejected_count': 0, + 'psf_quality_keep_mask': quality_mask, + 'ensemble_frame_keep_mask': quality_mask, + }, + { + 'key': 'comp2', 'comp_index': 1, 'label': 'Comp 2', 'position': [30, 40], + 'aggregate_score': 0.002, 'coverage_rejected': False, + 'suitability_outlier_rejected': False, 'overexposure_rejected_count': 0, + 'psf_quality_keep_mask': quality_mask, + 'ensemble_frame_keep_mask': quality_mask, + }, + ] + comparison_calibration = { + 'method': 'aperture', + 'method_label': 'Aperture photometry (aper=5px, annulus=10px)', + 'a': 0, + 'an': 0, + 'aper': 5.0, + 'annulus': 10.0, + 'field_score': 0.0015, + 'field_image_keep_mask': quality_mask, + 'comp_summaries': comp_summaries, + } + psf_data = { + 'target': np.ones((frame_count, 7), dtype=float), + 'comp1': np.ones((frame_count, 7), dtype=float), + 'comp2': np.ones((frame_count, 7), dtype=float), + } + aper_data = { + 'target': target_flux[:, None, None], + 'target_unc': np.full((frame_count, 1, 1), 1.0), + 'comp1': comp1_flux[:, None, None], + 'comp1_unc': np.full((frame_count, 1, 1), 1.0), + 'comp2': comp2_flux[:, None, None], + 'comp2_unc': np.full((frame_count, 1, 1), 1.0), + } + calibration_stars = { + 'C1': { + 'pos': [10, 20], 'mag': 12.0, 'error': 0.01, 'mag_band': 'V', + 'catalog_source': 'Synthetic catalog', + }, + 'C2': { + 'pos': [30, 40], 'mag': 12.0 + 2.5 * np.log10(2.0), + 'error': 0.011, 'mag_band': 'V', 'catalog_source': 'Synthetic catalog', + }, + } + + result = exotic_module.select_stellar_variability_only_photometry( + times, + times, + np.ones(frame_count), + _stellar_variability_only_planet_dict(), + comparison_calibration, + psf_data, + aper_data, + target_flux, + use_ensemble_photometry=True, + maximum_number_of_ensemble_comparisons_for_stellar_variability=2, + calibration_stars=calibration_stars, + observed_filter='V', + ) + + selected = result['selected_result'] + assert result['selection_metric'] == 'stellar_variability_ensemble' + assert selected['comp_index'] is None + assert selected['ensemble_member_keys'] == ['comp1', 'comp2'] + assert selected['fit'].stellar_variability_ensemble_members + assert any( + 'Using a 2-star comparison ensemble for stellar-variability products only:' in message + for message in logged + ) + assert len(selected['fit'].stellar_variability_ensemble_magnitudes) == len(selected['fit'].time) + np.testing.assert_allclose( + selected['fit'].differential_magnitude_reference_flux, + 375.0, + atol=1.0e-10, + ) + differential_series = exotic_module.differential_magnitude_series_from_fit( + selected['fit'], + apply_airmass_correction=False, + ) + expected_differential = -2.5 * np.log10(target_flux / 375.0) + np.testing.assert_allclose( + differential_series['magnitude'], + expected_differential, + atol=1.0e-10, + ) + assert abs(float(np.nanmedian(differential_series['magnitude']))) > 0.5 + + # Final output preparation may refresh the normalized fitting photometry. + # That must never overwrite the separately retained raw ensemble reference. + exotic_module.annotate_stellar_variability_raw_photometry( + selected['fit'], + target_flux, + target_flux, + target_flux_error=np.ones(frame_count), + comp_flux_error=np.ones(frame_count), + ) + differential_after_fit_refresh = exotic_module.differential_magnitude_series_from_fit( + selected['fit'], + apply_airmass_correction=False, + ) + np.testing.assert_allclose( + differential_after_fit_refresh['magnitude'], + expected_differential, + atol=1.0e-10, + ) + + vsp_params = exotic_module.build_stellar_variability_params_from_photometry_selection( + result, + calibration_stars, + tmp_path, + 'Synthetic', + observed_filter='V', + observation_date='2024-01-02', + ) + aid_path = exotic_module.AIDOutputFiles( + selected['fit'], + {'sName': 'Synthetic', 'pName': 'Synthetic b'}, + { + 'save': tmp_path, + 'date': '2024-01-02', + 'aavso_num': 'TEST', + 'camera': 'CCD', + 'lat': 0.0, + 'long': 0.0, + 'elev': 0.0, + 'filter': 'V', + }, + 'AUID-TEST', + None, + vsp_params, + ).aavso() + + assert len(vsp_params) == frame_count + np.testing.assert_allclose( + [row['mag'] for row in vsp_params], + selected['fit'].stellar_variability_ensemble_magnitudes, + atol=1.0e-10, + ) + np.testing.assert_allclose( + [row['differential_mag'] for row in vsp_params], + expected_differential, + atol=1.0e-10, + ) + assert aid_path.is_file() + aid_text = aid_path.read_text(encoding='utf-8') + assert '#ENSEMBLE-COMPARISONS-XC=' in aid_text + assert 'AUID-TEST,' in aid_text + + +def test_stellar_variability_exact_comparisons_use_every_supplied_member_without_catalogue(): + frame_count = 8 + times = np.linspace(10.2, 10.3, frame_count) + quality_mask = np.ones(frame_count, dtype=bool) + target_flux = np.linspace(990.0, 1010.0, frame_count) + comparison_calibration = { + 'method': 'aperture', + 'method_label': 'Aperture photometry', + 'a': 0, + 'an': 0, + 'field_score': np.inf, + 'field_image_keep_mask': quality_mask, + 'comp_summaries': [ + { + 'key': 'comp1', 'comp_index': 0, 'label': 'Comp 1', 'position': [10, 20], + 'aggregate_score': np.inf, 'coverage_rejected': True, + 'suitability_outlier_rejected': True, + 'psf_quality_keep_mask': quality_mask, + 'ensemble_frame_keep_mask': quality_mask, + }, + { + 'key': 'comp2', 'comp_index': 1, 'label': 'Comp 2', 'position': [30, 40], + 'aggregate_score': np.inf, 'coverage_rejected': True, + 'suitability_outlier_rejected': True, + 'psf_quality_keep_mask': quality_mask, + 'ensemble_frame_keep_mask': quality_mask, + }, + ], + } + psf_data = { + 'target': np.ones((frame_count, 7), dtype=float), + 'comp1': np.ones((frame_count, 7), dtype=float), + 'comp2': np.ones((frame_count, 7), dtype=float), + } + aper_data = { + 'target': target_flux[:, None, None], + 'target_unc': np.ones((frame_count, 1, 1)), + 'comp1': np.full((frame_count, 1, 1), 500.0), + 'comp1_unc': np.ones((frame_count, 1, 1)), + 'comp2': np.full((frame_count, 1, 1), 250.0), + 'comp2_unc': np.ones((frame_count, 1, 1)), + } + + result = exotic_module.select_stellar_variability_only_photometry( + times, + times, + np.linspace(1.0, 1.5, frame_count), + _stellar_variability_only_planet_dict(), + comparison_calibration, + psf_data, + aper_data, + target_flux, + use_ensemble_photometry=True, + calibration_stars={}, + observed_filter='V', + require_apparent_magnitudes=False, + use_exactly_the_comps_provided=True, + ) + + selected = result['selected_result'] + assert result['selection_metric'] == 'exact_stellar_variability_ensemble' + assert selected['ensemble_member_keys'] == ['comp1', 'comp2'] + assert [ + member['key'] for member in selected['fit'].stellar_variability_ensemble_members + ] == ['comp1', 'comp2'] + assert np.all(np.isnan(selected['fit'].stellar_variability_ensemble_magnitudes)) + + +def test_stellar_variability_selector_opt_out_restores_single_comp_selection(): + frame_count = 24 + times = np.linspace(10.2, 10.3, frame_count) + quality_mask = np.ones(frame_count, dtype=bool) + comparison_calibration = { + 'method': 'aperture', + 'method_label': 'Aperture photometry', + 'a': 0, + 'an': 0, + 'aper': 5.0, + 'annulus': 10.0, + 'field_score': 0.001, + 'field_image_keep_mask': quality_mask, + 'comp_summaries': [{ + 'key': 'comp1', 'comp_index': 0, 'label': 'Comp 1', 'position': [10, 20], + 'aggregate_score': 0.001, 'coverage_rejected': False, + 'suitability_outlier_rejected': False, 'overexposure_rejected_count': 0, + 'psf_quality_keep_mask': quality_mask, + 'ensemble_frame_keep_mask': quality_mask, + }], + } + target_flux = 1000.0 * (1.0 + np.linspace(-0.001, 0.001, frame_count)) + comp_flux = np.full(frame_count, 500.0) + psf_data = { + 'target': np.ones((frame_count, 7), dtype=float), + 'comp1': np.ones((frame_count, 7), dtype=float), + } + aper_data = { + 'target': target_flux[:, None, None], + 'target_unc': np.full((frame_count, 1, 1), 1.0), + 'comp1': comp_flux[:, None, None], + 'comp1_unc': np.full((frame_count, 1, 1), 1.0), + } + + result = exotic_module.select_stellar_variability_only_photometry( + times, + times, + np.ones(frame_count), + _stellar_variability_only_planet_dict(), + comparison_calibration, + psf_data, + aper_data, + target_flux, + use_ensemble_photometry=False, + ) + + assert result['selection_metric'] == 'stellar_variability_scatter' + assert result['selected_result']['comp_index'] == 0 diff --git a/tests/test_nonlinear_ld.py b/tests/test_nonlinear_ld.py new file mode 100644 index 00000000..79f493e8 --- /dev/null +++ b/tests/test_nonlinear_ld.py @@ -0,0 +1,184 @@ +import pytest + +import exotic.exotic as exotic_module + + +def test_nonlinear_ld_non_interactive_treats_g_as_photographic_g_without_prompt(monkeypatch): + ld = exotic_module.LimbDarkening({}) + monkeypatch.setattr(ld, "calculate_ld", lambda: None) + monkeypatch.setattr( + exotic_module, + "user_input", + lambda *_args, **_kwargs: pytest.fail("recognized G filter must not prompt"), + ) + info_dict = { + "filter": "G", + "wl_min": None, + "wl_max": None, + "ld_uncertainties": "y", + } + + exotic_module.nonlinear_ld(ld, info_dict, non_interactive_run=True) + + assert info_dict["filter"] == "PG" + assert info_dict["filter_desc"] == "Photographic G" + assert info_dict["wl_min"] == 502.8 + assert info_dict["wl_max"] == 586.8 + + +def test_nonlinear_ld_non_interactive_uses_neutral_cbb_name_without_prompt(monkeypatch): + ld = exotic_module.LimbDarkening({}) + monkeypatch.setattr(ld, "calculate_ld", lambda: None) + monkeypatch.setattr( + exotic_module, + "user_input", + lambda *_args, **_kwargs: pytest.fail("recognized CBB filter must not prompt"), + ) + info_dict = { + "filter": "CBB", + "wl_min": None, + "wl_max": None, + "ld_uncertainties": "y", + } + + exotic_module.nonlinear_ld(ld, info_dict, non_interactive_run=True) + + assert info_dict["filter"] == "CBB" + assert info_dict["filter_desc"] == "CBB" + assert info_dict["wl_min"] == 500.0 + assert info_dict["wl_max"] == 1000.0 + + +def test_nonlinear_ld_non_interactive_treats_cv_as_clearv_without_prompt(monkeypatch): + ld = exotic_module.LimbDarkening({}) + monkeypatch.setattr(ld, "calculate_ld", lambda: None) + monkeypatch.setattr( + exotic_module, + "user_input", + lambda *_args, **_kwargs: pytest.fail("recognized CV filter must not prompt"), + ) + info_dict = { + "filter": "CV", + "wl_min": None, + "wl_max": None, + "ld_uncertainties": "y", + } + + exotic_module.nonlinear_ld(ld, info_dict, non_interactive_run=True) + + assert info_dict["filter"] == "CV" + assert info_dict["filter_desc"] == "CV" + assert info_dict["wl_min"] == 350.0 + assert info_dict["wl_max"] == 1000.0 + + +class UnrecognizedFilterLimbDarkening: + fwhm_names_nonspecific = {} + + @staticmethod + def check_fwhm(_observed_filter): + return False + + @staticmethod + def check_standard(_observed_filter): + return False + + +class BooleanOptionLimbDarkening(UnrecognizedFilterLimbDarkening): + filter_name = None + filter_desc = None + wl_min = None + wl_max = None + + def calculate_ld(self): + return None + + +def set_boolean_option_filter(ld, label): + ld.filter_name = label + ld.filter_desc = label + ld.wl_min = 400.0 + ld.wl_max = 700.0 + + +@pytest.mark.parametrize("config_value", [True, 1, "1", "y", "Y", "yes", "TRUE", "on"]) +def test_nonlinear_ld_boolean_option_accepts_true_forms(monkeypatch, config_value): + ld = BooleanOptionLimbDarkening() + monkeypatch.setattr( + exotic_module, + "user_input", + lambda prompt, **_kwargs: 1 if "enter 1" in prompt.lower() else pytest.fail( + "valid true boolean must not trigger the y/n prompt" + ), + ) + monkeypatch.setattr( + exotic_module, + "standard_filter", + lambda selected_ld, _observed_filter: set_boolean_option_filter(selected_ld, "standard"), + ) + monkeypatch.setattr( + exotic_module, + "user_entered_ld", + lambda *_args, **_kwargs: pytest.fail("true must select calculated limb darkening"), + ) + info_dict = { + "filter": "mystery-band", + "wl_min": None, + "wl_max": None, + "ld_uncertainties": config_value, + } + + exotic_module.nonlinear_ld(ld, info_dict) + + assert info_dict["filter"] == "standard" + + +@pytest.mark.parametrize("config_value", [False, 0, "0", "n", "N", "no", "FALSE", "off"]) +def test_nonlinear_ld_boolean_option_accepts_false_forms(monkeypatch, config_value): + ld = BooleanOptionLimbDarkening() + monkeypatch.setattr( + exotic_module, + "user_input", + lambda *_args, **_kwargs: pytest.fail("valid false boolean must not prompt"), + ) + monkeypatch.setattr( + exotic_module, + "standard_filter", + lambda *_args, **_kwargs: pytest.fail("false must select user-entered limb darkening"), + ) + monkeypatch.setattr( + exotic_module, + "user_entered_ld", + lambda selected_ld, _observed_filter: set_boolean_option_filter(selected_ld, "manual"), + ) + info_dict = { + "filter": "mystery-band", + "wl_min": None, + "wl_max": None, + "ld_uncertainties": config_value, + } + + exotic_module.nonlinear_ld(ld, info_dict) + + assert info_dict["filter"] == "manual" + + +def test_nonlinear_ld_non_interactive_rejects_unrecognized_filter_without_prompt(monkeypatch): + monkeypatch.setattr( + exotic_module, + "user_input", + lambda *_args, **_kwargs: pytest.fail("non-interactive limb-darkening selection must not prompt"), + ) + info_dict = { + "filter": "mystery-band", + "wl_min": None, + "wl_max": None, + "ld_uncertainties": "y", + } + + with pytest.raises(ValueError, match="did not recognize the filter 'mystery-band'"): + exotic_module.nonlinear_ld( + UnrecognizedFilterLimbDarkening(), + info_dict, + non_interactive_run=True, + ) diff --git a/tests/test_output_files.py b/tests/test_output_files.py new file mode 100644 index 00000000..18c6ad52 --- /dev/null +++ b/tests/test_output_files.py @@ -0,0 +1,2334 @@ +import json +from types import SimpleNamespace + +import numpy as np +import pytest + +from exotic.output_files import ( + AREA_DEPTH_LABEL, + OBSERVABLE_DEPTH_DELTA_LABEL, + OBSERVABLE_DEPTH_LABEL, + PRIOR_OBSERVABLE_DEPTH_LABEL, + AIDOutputFiles, + OutputFiles, + aavso_detrend_model, + aid_comparison_coordinate_headers, + aavso_dicts, + aavso_undetrended_flux_series, + build_aavso_qc_metadata, + fit_empirical_transit_uncertainty, + fit_impact_parameter_value_error, + differential_magnitude_series_from_fit, + magnitude_series_from_fit, + save_comp_star_calibration_summary, + write_differential_magnitude_csv, +) +from exotic.transit_depth import ( + fit_transit_depth_summary, + observable_depth_percent, + radius_ratio_area_depth_percent, +) + + +def test_stellar_variability_differential_magnitudes_never_apply_airmass_correction(): + fit = SimpleNamespace( + stellar_variability_only=True, + time=np.array([2460000.1, 2460000.2, 2460000.3]), + data=np.ones(3), + dataerr=np.full(3, 0.01), + airmass=np.array([1.1, 1.3, 1.5]), + airmass_model=np.array([0.8, 1.0, 1.2]), + transit=np.ones(3), + stellar_variability_target_flux=np.array([80.0, 100.0, 120.0]), + stellar_variability_comp_flux=np.full(3, 100.0), + stellar_variability_target_flux_error=np.ones(3), + stellar_variability_comp_flux_error=np.ones(3), + ) + + series = differential_magnitude_series_from_fit(fit) + + np.testing.assert_allclose( + series['magnitude'], + -2.5 * np.log10(fit.stellar_variability_target_flux / fit.stellar_variability_comp_flux), + ) + assert series['airmass_corrected'] is False + + +def test_differential_csv_does_not_require_apparent_magnitude_calibration(tmp_path): + fit = SimpleNamespace( + stellar_variability_only=True, + time=np.array([2460000.1, 2460000.2]), + data=np.ones(2), + dataerr=np.full(2, 0.01), + airmass=np.array([1.1, 1.2]), + airmass_model=np.array([0.9, 1.1]), + transit=np.ones(2), + stellar_variability_target_flux=np.array([500.0, 550.0]), + stellar_variability_comp_flux=np.array([1000.0, 1000.0]), + stellar_variability_target_flux_error=np.full(2, 2.0), + stellar_variability_comp_flux_error=np.full(2, 3.0), + ) + + output_path = write_differential_magnitude_csv( + fit, + tmp_path, + 'Variable Star', + observation_date='2026-08-02', + observed_filter='V', + ) + + output_text = output_path.read_text(encoding='utf-8') + assert '# AIRMASS_CORRECTION=NO' in output_text + assert 'Differential Magnitude' in output_text + assert 'Apparent' not in output_text + assert ', 0.7526, 0.0054, 0.7526, 0.0054, 1.0000000, V, ' in output_text + assert ', 0.6491, 0.0051, 0.6491, 0.0051, 1.0000000, V, ' in output_text + + +class DummyFit: + def __init__(self): + self.parameters = { + "tmid": 2450000.123456, + "rprs": 0.1234, + "ars": 12.0, + "per": 2.15, + "inc": 88.5, + "ecc": 0.0, + "omega": 90.0, + "u0": 0.0, + "u1": 0.0, + "u2": 0.0, + "u3": 0.0, + "a1": 1.0, + "a2": 0.0, + } + self.errors = { + "tmid": 0.0001, + "rprs": 0.001, + "ars": 0.4, + "inc": 0.2, + "a1": 0.1, + "a2": 0.1, + } + self.time = [2450000.123456] + self.time_upsample = np.linspace(2450000.0, 2450000.2, 128) + self.data = [1.0] + self.dataerr = [0.01] + self.residuals = 0.01 + self.airmass_model = [1.0] + self.transit = [1.0 - self.parameters["rprs"] ** 2] + self.transit_upsample = np.ones_like(self.time_upsample) + self.transit_upsample[64] = 1.0 - self.parameters["rprs"] ** 2 + self.prior = { + "tmid": 2450000.123456, + "rprs": 0.1, + "ars": 12.0, + "per": 2.15, + "inc": 88.5, + "ecc": 0.0, + "omega": 90.0, + "u0": 0.0, + "u1": 0.0, + "u2": 0.0, + "u3": 0.0, + } + + +def test_prior_depth_uses_available_gj436_geometry(): + fit = DummyFit() + prior = { + "Published Mid-Transit Time": 2454510.80162, + "Rp/Rs": 0.0822, + "a/Rs": 13.73, + "Orbital Period (days)": 2.64388312, + "Orbital Inclination (deg)": 86.44, + "Orbital Eccentricity": 0.13827, + "Argument of Periastron (deg)": 351.0, + "u0": 0.0, + "u1": 0.0, + "u2": 0.0, + "u3": 0.0, + } + + summary = fit_transit_depth_summary( + fit, + prior_parameters=prior, + prior_errors={ + "Rp/Rs Uncertainty": 0.001, + "a/Rs Uncertainty": 0.46, + "Orbital Inclination Uncertainty": 0.17, + }, + ) + + assert summary["prior_observable_depth"] == pytest.approx(0.675684, abs=1.0e-5) + assert summary["prior_observable_depth_error"] == pytest.approx(0.01644, abs=1.0e-6) + + +def test_prior_depth_respects_inclination_for_non_transiting_geometry(): + fit = DummyFit() + prior = { + "tmid": 2454510.80162, + "rprs": 0.0822, + "ars": 13.73, + "per": 2.64388312, + "inc": 0.0, + "ecc": 0.13827, + "omega": 351.0, + "u0": 0.0, + "u1": 0.0, + "u2": 0.0, + "u3": 0.0, + } + + summary = fit_transit_depth_summary(fit, prior_parameters=prior) + + assert summary["prior_observable_depth"] == pytest.approx(0.0) + + +def test_final_params_writes_stellar_variability_only_payload(tmp_path): + (tmp_path / "working_artifacts").mkdir() + fit = SimpleNamespace( + stellar_variability_only=True, + time=np.arange(6, dtype=float), + stellar_variability_scatter=0.00123, + stellar_variability_transit_exclusion={ + 'rejected_point_count': 2, + 'duration_days': 0.083, + 'note': 'Excluded synthetic transit-window points.', + }, + airmass_fit_skipped=True, + airmass_correction_note=( + "Skipped in stellar-variability-only mode; no transit/systematics model was fit." + ), + ) + p_dict = {'pName': 'Syntheticb'} + i_dict = {'save': str(tmp_path), 'date': '2020-01-01'} + + OutputFiles(fit, p_dict, i_dict, [0.083]).final_planetary_params( + phot_opt=True, + vsp_params=None, + comp_star=2, + comp_coords=[10, 20], + min_aper=0, + min_annul=15, + photometry_info={'noise_budget_summary': 'gain only'}, + publish_to_root=True, + ) + + temp_file = next((tmp_path / "working_artifacts").glob("FinalParams_Syntheticb_2020-01-01.json")) + root_file = tmp_path / temp_file.name + payload = json.loads(temp_file.read_text()) + + params = payload["FINAL STELLAR VARIABILITY PARAMETERS"] + assert params["Analysis Mode"] == "Stellar variability only" + assert params["Transit model fitting"] == "Skipped" + assert params["Predicted in-transit points excluded"] == "2" + assert params["Residual scatter around flat stellar-variability model"] == "0.1230 %" + assert params["Stellar Variability Reference Star"] == "#2 - [10, 20]" + assert params["Optimal Method"] == "PSF photometry" + assert root_file.exists() + + +def test_final_params_describes_calibrated_stellar_variability_ensemble(tmp_path): + (tmp_path / "working_artifacts").mkdir() + fit = SimpleNamespace( + stellar_variability_only=True, + time=np.arange(3, dtype=float), + stellar_variability_scatter=0.001, + stellar_variability_transit_exclusion={'rejected_point_count': 0}, + ) + vsp_params = [{ + 'time': 2460000.1, + 'mag': 12.3, + 'mag_err': 0.01, + 'cname': 'ENSEMBLE (2 stars)', + 'ensemble_reference': True, + 'ensemble_member_count': 2, + 'ensemble_member_labels': ['C1', 'C2'], + 'mag_band': 'V', + }] + p_dict = {'pName': 'Syntheticb'} + i_dict = {'save': str(tmp_path), 'date': '2020-01-01'} + + OutputFiles(fit, p_dict, i_dict, []).final_planetary_params( + phot_opt=True, + vsp_params=vsp_params, + comp_star='ensemble', + comp_coords=None, + min_aper=0, + min_annul=15, + ) + + output_path = next((tmp_path / "working_artifacts").glob("FinalParams_Syntheticb_2020-01-01.json")) + params = json.loads(output_path.read_text())["FINAL STELLAR VARIABILITY PARAMETERS"] + assert params["Stellar Variability Reference Star"] == "ensemble" + assert params["Variable Reference Star"] == ( + "Calibrated comparison-star ensemble (2 stars): C1, C2" + ) + assert "calibrated comparison-star ensemble" in params["Variable Reference Measurement"] + + +def test_final_lightcurve_writes_stellar_variability_magnitudes(tmp_path): + (tmp_path / "working_artifacts").mkdir() + fit = SimpleNamespace( + stellar_variability_only=True, + stellar_variability_params=[ + { + "time": 2461229.89899, + "mag": 13.7378, + "mag_err": 0.0042, + "differential_mag": 1.2378, + "differential_mag_err": 0.0021, + "mag_band": "r", + "airmass": 1.193135, + }, + { + "time": 2461229.90109, + "mag": 13.7401, + "mag_err": 0.0044, + "differential_mag": 1.2401, + "differential_mag_err": 0.0022, + "mag_band": "r", + "airmass": 1.1984942, + }, + ], + ) + p_dict = {'pName': 'WASP-194 b', 'sName': 'WASP-194'} + i_dict = {'save': str(tmp_path), 'date': '2026-07-08', 'filter': 'SR'} + + OutputFiles(fit, p_dict, i_dict, []).final_lightcurve(np.array([])) + + output_text = next((tmp_path / "working_artifacts").glob("FinalLightCurve_WASP-194b_2026-07-08.csv")).read_text() + + assert "# FINAL STELLAR VARIABILITY TIMESERIES OF WASP-194" in output_text + assert "Apparent Magnitude,Apparent Magnitude Uncertainty" in output_text + assert "Raw Differential Magnitude,Raw Differential Magnitude Uncertainty" in output_text + assert "2461229.89899, 13.7378, 0.0042, 1.2378, 0.0021, r, 1.193135" in output_text + assert "Flux" not in output_text + + +def test_final_lightcurve_adds_transit_apparent_magnitude_columns_when_calibrated(tmp_path): + (tmp_path / "working_artifacts").mkdir() + fit = SimpleNamespace( + time=np.array([2461229.9, 2461229.91]), + detrended=np.array([1.0, 0.99]), + dataerr=np.array([0.001, 0.001]), + airmass_model=np.ones(2), + transit=np.array([1.0, 0.99]), + stellar_variability_params=[ + {"time": 2461229.9, "mag": 13.739, "mag_err": 0.001, "mag_band": "r"}, + {"time": 2461229.91, "mag": 13.741, "mag_err": 0.002, "mag_band": "r"}, + ], + ) + p_dict = {'pName': 'WASP-194 b', 'sName': 'WASP-194'} + i_dict = {'save': str(tmp_path), 'date': '2026-07-08', 'filter': 'SR'} + + OutputFiles(fit, p_dict, i_dict, []).final_lightcurve(np.array([0.1, 0.2])) + + output_text = next((tmp_path / "working_artifacts").glob("FinalLightCurve_WASP-194b_2026-07-08.csv")).read_text() + + assert "Raw Differential Magnitude,Raw Differential Magnitude Uncertainty" in output_text + assert "Corrected Differential Magnitude,Corrected Differential Magnitude Uncertainty" in output_text + assert "Apparent Magnitude,Apparent Magnitude Uncertainty,Band" in output_text + assert ( + "2461229.9, 0.1, 1.0, 0.001, 1.0, na, 1.0, " + "-0.0000, 0.0011, -0.0000, 0.0011, 13.7400" + in output_text + ) + assert output_text.rstrip().endswith(", r") + + +def test_final_lightcurve_keeps_differential_magnitude_when_apparent_calibration_is_unavailable(tmp_path): + (tmp_path / "working_artifacts").mkdir() + fit = SimpleNamespace( + time=np.array([2461229.9]), + data=np.array([0.8]), + detrended=np.array([0.8]), + dataerr=np.array([0.008]), + airmass_model=np.ones(1), + transit=np.ones(1), + ) + p_dict = {'pName': 'Uncalibrated b', 'sName': 'Uncalibrated'} + i_dict = {'save': str(tmp_path), 'date': '2026-07-08', 'filter': 'V'} + + OutputFiles(fit, p_dict, i_dict, []).final_lightcurve(np.array([0.1])) + + output_text = next( + (tmp_path / "working_artifacts").glob("FinalLightCurve_Uncalibratedb_2026-07-08.csv") + ).read_text() + expected_differential = -2.5 * np.log10(0.8) + assert f"{expected_differential:.4f}" in output_text + assert ", na, na, V" in output_text + + +def test_magnitude_series_preserves_raw_ratio_for_later_apparent_recalibration(): + target_flux = np.array([500.0, 550.0]) + reference_flux = np.full(2, 1000.0) + target_error = np.full(2, 2.0) + reference_error = np.full(2, 3.0) + differential_mag = -2.5 * np.log10(target_flux / reference_flux) + magnitude_factor = 2.5 / np.log(10.0) + differential_error = magnitude_factor * np.sqrt( + (target_error / target_flux) ** 2 + + (reference_error / reference_flux) ** 2 + ) + fit = SimpleNamespace( + time=np.array([2461229.9, 2461229.91]), + data=np.array([1.0, 1.1]), + dataerr=np.full(2, 0.001), + detrended=np.array([1.0, 1.1]), + airmass=np.array([1.1, 1.2]), + airmass_model=np.ones(2), + transit=np.ones(2), + stellar_variability_target_flux=target_flux, + stellar_variability_comp_flux=reference_flux, + stellar_variability_target_flux_error=target_error, + stellar_variability_comp_flux_error=reference_error, + stellar_variability_params=[{ + "time": 2461229.9, + "mag": 12.0 + differential_mag[0], + "mag_err": np.hypot(0.02, differential_error[0]), + "differential_mag": differential_mag[0], + "differential_mag_err": differential_error[0], + "cmag": 12.0, + "cmag_err": 0.02, + "mag_band": "V", + }], + ) + + series = magnitude_series_from_fit(fit, apply_airmass_correction=False) + + np.testing.assert_allclose(series['differential_magnitude'], differential_mag) + np.testing.assert_allclose(series['differential_magnitude_error'], differential_error) + np.testing.assert_allclose(series['apparent_magnitude'], 12.0 + differential_mag) + np.testing.assert_allclose( + series['apparent_magnitude_error'], + np.hypot(0.02, differential_error), + ) + + +def test_magnitude_series_keeps_raw_and_airmass_corrected_differential_values(): + target_flux = np.array([800.0, 1000.0, 1200.0]) + reference_flux = np.full(3, 1000.0) + target_error = np.full(3, 2.0) + reference_error = np.full(3, 3.0) + airmass_model = np.array([0.8, 1.0, 1.2]) + relative_correction = airmass_model / np.median(airmass_model) + raw_ratio = target_flux / reference_flux + expected_raw = -2.5 * np.log10(raw_ratio) + expected_corrected = -2.5 * np.log10(raw_ratio / relative_correction) + fit = SimpleNamespace( + stellar_variability_only=False, + time=np.array([2461229.9, 2461229.91, 2461229.92]), + data=raw_ratio, + dataerr=np.full(3, 0.001), + detrended=raw_ratio / relative_correction, + detrendederr=np.full(3, 0.001) / relative_correction, + airmass=np.array([1.1, 1.2, 1.3]), + airmass_model=airmass_model, + transit=np.ones(3), + stellar_variability_target_flux=target_flux, + stellar_variability_comp_flux=reference_flux, + stellar_variability_target_flux_error=target_error, + stellar_variability_comp_flux_error=reference_error, + ) + + series = magnitude_series_from_fit(fit) + + np.testing.assert_allclose(series['raw_differential_magnitude'], expected_raw) + np.testing.assert_allclose(series['corrected_differential_magnitude'], expected_corrected) + np.testing.assert_allclose(series['differential_magnitude'], expected_corrected) + np.testing.assert_allclose( + series['raw_differential_magnitude_error'], + series['corrected_differential_magnitude_error'], + ) + np.testing.assert_allclose( + series['differential_magnitude_correction_factor'], + relative_correction, + ) + assert series['correction_type'] == 'airmass' + assert series['correction_applied'] is True + assert series['airmass_corrected'] is True + + +def test_differential_csv_writes_raw_and_corrected_differential_rows(tmp_path): + fit = SimpleNamespace( + stellar_variability_only=False, + time=np.array([2461229.9, 2461229.91]), + data=np.array([0.8, 1.2]), + dataerr=np.full(2, 0.008), + detrended=np.ones(2), + detrendederr=np.full(2, 0.01), + airmass=np.array([1.1, 1.4]), + airmass_model=np.array([0.8, 1.2]), + transit=np.ones(2), + stellar_variability_target_flux=np.array([800.0, 1200.0]), + stellar_variability_comp_flux=np.full(2, 1000.0), + stellar_variability_target_flux_error=np.full(2, 2.0), + stellar_variability_comp_flux_error=np.full(2, 3.0), + ) + + output_path = write_differential_magnitude_csv( + fit, + tmp_path, + 'Transit Target', + observation_date='2026-08-17', + observed_filter='V', + ) + output_lines = output_path.read_text(encoding='utf-8').splitlines() + + assert '# AIRMASS_CORRECTION=YES' in output_lines + assert '# DIFFERENTIAL_MAGNITUDE_CORRECTION=airmass' in output_lines + assert 'Raw Differential Magnitude' in output_lines[2] + assert 'Corrected Differential Magnitude' in output_lines[2] + first_row = [value.strip() for value in output_lines[3].split(',')] + assert first_row[2] == f"{-2.5 * np.log10(0.8):.4f}" + assert first_row[4] == f"{-2.5 * np.log10(0.8 / 0.8):.4f}" + assert first_row[6] == '0.8000000' + + +def test_final_lightcurve_csv_writes_raw_and_corrected_differential_rows(tmp_path): + (tmp_path / 'working_artifacts').mkdir() + fit = SimpleNamespace( + stellar_variability_only=False, + time=np.array([2461229.9, 2461229.91]), + data=np.array([0.8, 1.2]), + dataerr=np.full(2, 0.008), + detrended=np.ones(2), + detrendederr=np.full(2, 0.01), + airmass=np.array([1.1, 1.4]), + airmass_model=np.array([0.8, 1.2]), + transit=np.ones(2), + stellar_variability_target_flux=np.array([800.0, 1200.0]), + stellar_variability_comp_flux=np.full(2, 1000.0), + stellar_variability_target_flux_error=np.full(2, 2.0), + stellar_variability_comp_flux_error=np.full(2, 3.0), + ) + p_dict = {'pName': 'Transit Target b', 'sName': 'Transit Target'} + i_dict = {'save': str(tmp_path), 'date': '2026-08-17', 'filter': 'V'} + + OutputFiles(fit, p_dict, i_dict, []).final_lightcurve(np.array([0.1, 0.2])) + output_path = next( + (tmp_path / 'working_artifacts').glob('FinalLightCurve_TransitTargetb_2026-08-17.csv') + ) + output_lines = output_path.read_text(encoding='utf-8').splitlines() + + assert '# DIFFERENTIAL_MAGNITUDE_CORRECTION=airmass' in output_lines + assert 'Raw Differential Magnitude' in output_lines[4] + assert 'Corrected Differential Magnitude' in output_lines[4] + first_row = [value.strip() for value in output_lines[5].split(',')] + assert first_row[5] == '1.1' + assert first_row[6] == '0.8' + assert first_row[7] == f"{-2.5 * np.log10(0.8):.4f}" + assert first_row[9] == f"{-2.5 * np.log10(0.8 / 0.8):.4f}" + + +def test_linear_baseline_metadata_reconstructs_raw_flux_and_differential_magnitude(): + times = np.array([2461229.9, 2461229.91, 2461229.92]) + corrected_flux = np.array([1.0, 0.99, 1.0]) + corrected_error = np.full(3, 0.01) + fit = SimpleNamespace( + stellar_variability_only=False, + time=times, + data=corrected_flux, + dataerr=corrected_error, + detrended=corrected_flux, + detrendederr=corrected_error, + airmass=np.array([1.1, 1.2, 1.3]), + transit=np.ones(3), + oot_baseline_detrending_applied=True, + oot_baseline_reference_time_bjd_tdb=times[1], + oot_baseline_intercept=1.0, + oot_baseline_slope=2.0, + ) + expected_model = 1.0 + 2.0 * (times - times[1]) + + detrend_model = aavso_detrend_model(fit) + raw_flux, raw_error = aavso_undetrended_flux_series(fit, detrend_model) + series = magnitude_series_from_fit(fit) + + np.testing.assert_allclose(detrend_model, expected_model) + np.testing.assert_allclose(raw_flux, corrected_flux * expected_model) + np.testing.assert_allclose(raw_error, corrected_error * expected_model) + np.testing.assert_allclose( + series['raw_differential_magnitude'], + -2.5 * np.log10(corrected_flux * expected_model), + ) + np.testing.assert_allclose( + series['corrected_differential_magnitude'], + -2.5 * np.log10(corrected_flux), + ) + assert series['correction_type'] == 'out_of_transit_linear_baseline' + assert series['correction_applied'] is True + assert series['airmass_corrected'] is False + + +def test_calibrated_ensemble_keeps_apparent_magnitudes_independent_of_raw_differential(): + target_flux = np.array([1000.0, 1010.0]) + raw_reference_flux = np.full(2, 375.0) + calibrated_magnitude = np.array([11.25, 11.27]) + calibrated_error = np.array([0.02, 0.021]) + expected_differential = -2.5 * np.log10(target_flux / raw_reference_flux) + fit = SimpleNamespace( + stellar_variability_only=True, + time=np.array([2461229.9, 2461229.91]), + data=np.ones(2), + detrended=np.ones(2), + dataerr=np.full(2, 0.001), + airmass=np.array([1.1, 1.2]), + airmass_model=np.ones(2), + transit=np.ones(2), + # The calibrated ensemble's normalized fitting reference remains + # separate from the raw instrumental reference used for DIFFMAG. + stellar_variability_target_flux=target_flux, + stellar_variability_comp_flux=target_flux.copy(), + stellar_variability_target_flux_error=np.ones(2), + stellar_variability_comp_flux_error=np.ones(2), + differential_magnitude_target_flux=target_flux, + differential_magnitude_reference_flux=raw_reference_flux, + differential_magnitude_target_flux_error=np.ones(2), + differential_magnitude_reference_flux_error=np.ones(2), + stellar_variability_ensemble_magnitudes=calibrated_magnitude, + stellar_variability_ensemble_magnitude_errors=calibrated_error, + stellar_variability_params=[ + { + 'time': 2461229.9, + 'mag': calibrated_magnitude[0], + 'mag_err': calibrated_error[0], + 'differential_mag': expected_differential[0], + 'differential_mag_err': 0.002, + 'mag_band': 'V', + }, + { + 'time': 2461229.91, + 'mag': calibrated_magnitude[1], + 'mag_err': calibrated_error[1], + 'differential_mag': expected_differential[1], + 'differential_mag_err': 0.002, + 'mag_band': 'V', + }, + ], + ) + + series = magnitude_series_from_fit(fit, apply_airmass_correction=False) + + np.testing.assert_allclose(series['differential_magnitude'], expected_differential) + np.testing.assert_allclose(series['apparent_magnitude'], calibrated_magnitude) + np.testing.assert_allclose(series['apparent_magnitude_error'], calibrated_error) + assert series['apparent_calibrated'] is True + + +def aavso_json_header(output_text, header_name): + prefix = f"#{header_name}=" + for line in output_text.splitlines(): + if line.startswith(prefix): + return json.loads(line[len(prefix):]) + raise AssertionError(f"Missing {header_name} header") + + +def test_observable_depth_is_separate_from_area_depth_for_grazing_geometry(): + parameters = { + "tmid": 0.0, + "rprs": 0.2, + "per": 3.0, + "ars": 10.0, + "inc": np.degrees(np.arccos(1.1 / 10.0)), + "ecc": 0.0, + "omega": 90.0, + "u0": 0.0, + "u1": 0.0, + "u2": 0.0, + "u3": 0.0, + } + errors = {"rprs": 0.01, "ars": 0.1, "inc": 0.1} + + area_depth, area_error = radius_ratio_area_depth_percent(parameters["rprs"], errors["rprs"]) + observable_depth, observable_error = observable_depth_percent(parameters, errors) + + assert area_depth == pytest.approx(4.0) + assert area_error == pytest.approx(0.4) + assert 0.0 < observable_depth < area_depth + assert observable_error > 0.0 + + +def test_aavso_output_includes_observatory_location_headers(tmp_path): + fit = DummyFit() + fit.oot_baseline_detrending_applied = True + fit.oot_baseline_detrending_note = "Applied weighted linear out-of-transit baseline detrending." + fit.oot_baseline_reference_time_bjd_tdb = fit.time[0] + fit.oot_baseline_intercept = 1.2 + fit.oot_baseline_slope = 0.05 + fit.oot_baseline_pre_points = 4 + fit.oot_baseline_post_points = 5 + fit.stellar_variability_target_flux = np.array([500.0]) + fit.stellar_variability_comp_flux = np.array([1000.0]) + fit.stellar_variability_target_flux_error = np.array([2.0]) + fit.stellar_variability_comp_flux_error = np.array([3.0]) + differential_mag = float(-2.5 * np.log10(0.5)) + differential_error = float( + (2.5 / np.log(10.0)) * np.hypot(2.0 / 500.0, 3.0 / 1000.0) + ) + fit.stellar_variability_params = [{ + "time": fit.time[0], + "mag": 12.0 + differential_mag, + "mag_err": np.hypot(0.02, differential_error), + "differential_mag": differential_mag, + "differential_mag_err": differential_error, + "cmag": 12.0, + "cmag_err": 0.02, + "mag_band": "V", + }] + p_dict = { + "pName": "HAT-P-32b", + "sName": "HAT-P-32", + "pPer": 2.1500082, + "pPerUnc": 1.3e-07, + "rprs": 0.1488623525, + "rprsUnc": 0.0005539487, + "aRs": 5.344, + "aRsUnc": 0.03949, + "inc": 88.98, + "incUnc": 0.7602, + "ecc": 0.159, + "dist": 245.7, + "pm_ra": 14.25, + "pm_dec": -9.5, + } + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "aavso_num": "RTZ", + "second_obs": "", + "obs_name": "Whipple Observatory", + "camera": "CCD", + "pixel_bin": "1x1", + "exposure": 60.0, + "lat": "+32.41638889", + "long": "-110.73444444", + "elev": 2616, + "notes": "na", + "filter": "CV", + "filter_desc": "Clear with V zero-point", + "wl_min": None, + "wl_max": None, + } + final_plot_source = tmp_path / "FinalLightCurve_HAT-P-32b_2020-01-01.png" + final_plot_source.write_bytes(b"final lightcurve") + diagnostics_dir = tmp_path / "Diagnostics" + diagnostics_dir.mkdir() + diagnostic_sources = [ + diagnostics_dir / filename + for filename in ( + "FinalTriangle_HAT-P-32b_2020-01-01.png", + "Triangle_HAT-P-32b_2020-01-01.png", + "ZoomedTrianglePlot_HAT-P-32b_2020-01-01.png", + "KTMF_QC_HAT-P-32b_2020-01-01.png", + "KTMF_QC_HAT-P-32b_2020-01-01.pdf", + "PriorPosteriorComparison_HAT-P-32b_2020-01-01.png", + "PriorPosteriorComparison_HAT-P-32b_2020-01-01.pdf", + ) + ] + for diagnostic_source in diagnostic_sources: + diagnostic_source.write_bytes(diagnostic_source.name.encode("utf-8")) + + OutputFiles(fit, p_dict, i_dict, [0.1]).aavso( + {"ra": "", "dec": "", "x": "493", "y": "202"}, + [1.0], + (0.1, 0.01), + (0.2, 0.01), + (0.3, 0.01), + (0.4, 0.01), + None, + ) + + output_file = tmp_path / "AAVSO_Files" / "AAVSO_HAT-P-32b_2020-01-01.txt" + output_text = output_file.read_text(encoding="utf-8") + assert ( + tmp_path / "AAVSO_Files" / final_plot_source.name + ).read_bytes() == b"final lightcurve" + for diagnostic_source in diagnostic_sources: + assert ( + tmp_path / "AAVSO_Files" / diagnostic_source.name + ).read_bytes() == diagnostic_source.name.encode("utf-8") + + assert "#OBSDATE=2020-01-01" in output_text + assert "#EXOPLANET_NAME=HAT-P-32 b" in output_text + assert "#OBSNAME=Whipple Observatory" in output_text + assert "#OBSLAT=+32.41638889" in output_text + assert "#OBSLON=-110.73444444" in output_text + assert "#OBSELEV=2616" in output_text + assert "#GAIADIST=245.7" in output_text + assert "#GAIAPMRA=14.25" in output_text + assert "#GAIAPMDEC=-9.5" in output_text + magnitude_fields = aavso_json_header(output_text, "MAGNITUDE_FIELDS-XC") + magnitude_row = aavso_json_header(output_text, "MAGNITUDE-XC") + baseline_metadata = aavso_json_header(output_text, "OUT_OF_TRANSIT_BASELINE-XC") + detrend_metadata = aavso_json_header(output_text, "DETREND_PARAMETERS-XC") + assert baseline_metadata == { + "applied": True, + "forward_correction": "detrended_flux = raw_flux / baseline(t)", + "intercept": 1.2, + "inverse_correction": "raw_flux = detrended_flux * baseline(t)", + "model": ( + "baseline(t) = intercept + slope_per_day * " + "(BJD_TDB - reference_time_bjd_tdb)" + ), + "note": "Applied weighted linear out-of-transit baseline detrending.", + "post_egress_point_count": 5, + "pre_ingress_point_count": 4, + "reference_time_bjd_tdb": fit.time[0], + "serialized_model_available": True, + "slope_per_day": 0.05, + } + assert "#DETREND_PARAMETERS=AIRMASS, AIRMASS CORRECTION FUNCTION" in output_text + assert detrend_metadata == { + "DETREND_1": "airmass", + "DETREND_2": "out_of_transit_linear_baseline_correction_function", + "standard_header_preserved": True, + } + assert magnitude_fields["apparent_calibrated"] is True + assert magnitude_fields["differential_magnitude"].startswith("target minus") + assert magnitude_fields["raw_differential_magnitude"].startswith("target minus") + assert magnitude_fields["differential_magnitude"].endswith( + "corrected_differential_magnitude" + ) + assert magnitude_row["raw_differential_magnitude"] == round(differential_mag, 4) + assert magnitude_row["raw_differential_magnitude_error"] == round(differential_error, 4) + assert magnitude_row["corrected_differential_magnitude"] == round(differential_mag, 4) + assert magnitude_row["corrected_differential_magnitude_error"] == round( + differential_error, + 4, + ) + assert magnitude_row["differential_magnitude"] == round(differential_mag, 4) + assert magnitude_row["differential_magnitude_error"] == round(differential_error, 4) + assert magnitude_row["apparent_magnitude"] == round(12.0 + differential_mag, 4) + assert magnitude_row["apparent_magnitude_error"] == round( + np.hypot(0.02, differential_error), + 4, + ) + assert "2450000.123456,1.2,0.012,1.0,1.2" in output_text + + +def test_aavso_output_omits_obsname_header_when_blank(tmp_path): + fit = DummyFit() + p_dict = { + "pName": "HAT-P-32 b", + "sName": "HAT-P-32", + "pPer": 2.1500082, + "pPerUnc": 1.3e-07, + "rprs": 0.1488623525, + "rprsUnc": 0.0005539487, + "aRs": 5.344, + "aRsUnc": 0.03949, + "inc": 88.98, + "incUnc": 0.7602, + "ecc": 0.159, + "dist": None, + "pm_ra": None, + "pm_dec": None, + } + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "aavso_num": "RTZ", + "second_obs": "", + "obs_name": "", + "camera": "CCD", + "pixel_bin": "1x1", + "exposure": 60.0, + "lat": "+32.41638889", + "long": "-110.73444444", + "elev": 2616, + "notes": "na", + "filter": "CV", + "filter_desc": "Clear with V zero-point", + "wl_min": None, + "wl_max": None, + } + + OutputFiles(fit, p_dict, i_dict, [0.1]).aavso( + {"ra": "", "dec": "", "x": "493", "y": "202"}, + [1.0], + (0.1, 0.01), + (0.2, 0.01), + (0.3, 0.01), + (0.4, 0.01), + None, + ) + + output_file = tmp_path / "AAVSO_Files" / "AAVSO_HAT-P-32b_2020-01-01.txt" + output_text = output_file.read_text(encoding="utf-8") + + assert "#OBSNAME=" not in output_text + assert "#GAIADIST=" not in output_text + assert "#GAIAPMRA=" not in output_text + assert "#GAIAPMDEC=" not in output_text + magnitude_fields = aavso_json_header(output_text, "MAGNITUDE_FIELDS-XC") + magnitude_row = aavso_json_header(output_text, "MAGNITUDE-XC") + assert magnitude_fields["apparent_calibrated"] is False + assert magnitude_row["differential_magnitude"] == pytest.approx(0.0) + assert magnitude_row["apparent_magnitude"] is None + assert magnitude_row["apparent_magnitude_error"] is None + + +def test_aid_comparison_coordinate_headers_index_unique_comparisons_on_separate_lines(): + headers = aid_comparison_coordinate_headers( + [ + {"cname": "Comp A", "comp_ra": 10.1, "comp_dec": -20.2}, + {"cname": "Comp A", "comp_ra": 10.1, "comp_dec": -20.2}, + {"cname": "Comp B", "comp_ra": 11.3, "comp_dec": -21.4}, + ], + indexed=True, + ) + + assert headers.splitlines() == [ + "#COMPARISON_1_NAME=Comp A", + "#COMPARISON_1_RA=10.1000000", + "#COMPARISON_1_DEC=-20.2000000", + "#COMPARISON_2_NAME=Comp B", + "#COMPARISON_2_RA=11.3000000", + "#COMPARISON_2_DEC=-21.4000000", + ] + + +def test_aid_output_includes_nextastro_comparison_metadata(tmp_path): + fit = DummyFit() + p_dict = { + "pName": "HAT-P-32 b", + "sName": "HAT-P-32", + } + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "aavso_num": "RTZ", + "camera": "CCD", + "filter": "V", + "lat": "+32.41638889", + "long": "-110.73444444", + "elev": 2616, + } + vsp_params = [{ + "time": 2450000.12345, + "mag": 12.34567, + "mag_err": 0.012345, + "differential_mag": 0.24567, + "differential_mag_err": 0.006789, + "airmass": 1.234, + "cname": "RA=10.1000000 Dec=-20.2000000", + "cmag": 12.1, + "cmag_err": 0.03, + "pos": [493, 202], + "comp_ra": 10.1, + "comp_dec": -20.2, + "catalog_ra": 10.10001, + "catalog_dec": -20.20001, + "catalog_source": "NextAstro photometry catalog", + "is_aavso_vsp": False, + "mag_band": "ClearV", + "catalog_mag_band": "V", + "source_id": 12345, + "separation_arcsec": 0.2, + }] + working_artifacts_dir = tmp_path / "working_artifacts" + working_artifacts_dir.mkdir() + finder_source = ( + working_artifacts_dir / "FOV_HAT-P-32b_LinearStretch_2020-01-01.png" + ) + finder_source.write_bytes(b"finder chart") + + AIDOutputFiles(fit, p_dict, i_dict, auid=None, chart_id=None, vsp_params=vsp_params).aavso() + + output_text = ( + tmp_path / "AAVSO_Files" / "AID_AAVSO_HAT-P-32_2020-01-01.txt" + ).read_text(encoding="utf-8") + assert ( + tmp_path / "AAVSO_Files" / finder_source.name + ).read_bytes() == b"finder chart" + metadata = aavso_json_header(output_text, "COMPARISON-CATALOG-XC") + + assert metadata["source"] == "NextAstro photometry catalog" + assert metadata["is_aavso_vsp"] is False + assert metadata["comparison_ra_deg"] == pytest.approx(10.1) + assert metadata["comparison_dec_deg"] == pytest.approx(-20.2) + assert metadata["apparent_magnitude"] == pytest.approx(12.1) + assert metadata["apparent_magnitude_error"] == pytest.approx(0.03) + assert metadata["magnitude_band"] == "V" + assert metadata["reported_measurement_band"] == "ClearV" + assert "#COMPARISON_RA=10.1000000\n#COMPARISON_DEC=-20.2000000\n" in output_text + assert "#DATE=BJD_TDB" in output_text + assert "HAT-P-32,2450000.12345,12.3457,0.0123,V,NO,STD" in output_text + assert ( + "#NAME,DATE,MAG,MERR,FILT,TRANS,MTYPE,CNAME,CMAG,KNAME,KMAG,AMASS," + "GROUP,CHART,NOTES,DIFFMAG,DIFFERR\n" + ) in output_text + aid_header = next(line for line in output_text.splitlines() if line.startswith("#NAME,")) + aid_data_row = next(line for line in output_text.splitlines() if not line.startswith("#")) + assert len(aid_header.split(",")) == len(aid_data_row.split(",")) == 17 + assert aid_data_row.split(",")[-3:] == ["na", "0.2457", "0.0068"] + assert "|DIFFMAG=" not in output_text + assert "|DIFFERR=" not in output_text + magnitude_fields = aavso_json_header(output_text, "MAGNITUDE_FIELDS-XC") + assert magnitude_fields["apparent_magnitude"] == "MAG" + assert magnitude_fields["differential_magnitude"] == "DIFFMAG" + assert magnitude_fields["differential_magnitude_error"] == "DIFFERR" + + +def test_aid_output_records_calibrated_ensemble_members(tmp_path): + fit = DummyFit() + p_dict = {"pName": "Target b", "sName": "Target"} + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "aavso_num": "RTZ", + "camera": "CCD", + "filter": "V", + "lat": "+32.4", + "long": "-110.7", + "elev": 2600, + } + vsp_params = [{ + "time": 2450000.12345, + "mag": 12.34, + "mag_err": 0.02, + "differential_mag": 1.234567, + "differential_mag_err": 0.00789, + "airmass": 1.234, + "cname": "ENSEMBLE (2 stars)", + "cmag": None, + "cmag_err": None, + "catalog_source": "Calibrated comparison-star ensemble", + "is_aavso_vsp": False, + "mag_band": "V", + "ensemble_reference": True, + "ensemble_member_count": 2, + "ensemble_member_labels": ["C1", "C2"], + "ensemble_member_positions": [[10, 20], [30, 40]], + "ensemble_member_catalog_magnitudes": [12.0, 12.5], + "ensemble_member_catalog_errors": [0.01, 0.011], + "ensemble_member_catalog_sources": ["Catalog A", "Catalog B"], + "ensemble_member_ra_degs": [10.1, 10.2], + "ensemble_member_dec_degs": [-20.1, -20.2], + "ensemble_member_catalog_colors": [0.5, 0.6], + "ensemble_member_catalog_color_labels": ["B-V", "B-V"], + "ensemble_member_color_deltas": [0.02, 0.08], + "ensemble_member_magnitude_deltas": [0.1, 0.4], + "ensemble_member_similarity_scores": [0.102, 0.408], + }] + + AIDOutputFiles(fit, p_dict, i_dict, auid=None, chart_id=None, vsp_params=vsp_params).aavso() + + output_text = ( + tmp_path / "AAVSO_Files" / "AID_AAVSO_Target_2020-01-01.txt" + ).read_text(encoding="utf-8") + metadata = aavso_json_header(output_text, "COMPARISON-CATALOG-XC") + ensemble_metadata = aavso_json_header(output_text, "ENSEMBLE-COMPARISONS-XC") + assert metadata["ensemble_reference"] is True + assert metadata["ensemble_member_count"] == 2 + assert metadata["ensemble_member_labels"] == ["C1", "C2"] + assert metadata["ensemble_member_ra_degs"] == [10.1, 10.2] + assert metadata["ensemble_member_dec_degs"] == [-20.1, -20.2] + assert metadata["ensemble_member_catalog_colors"] == [0.5, 0.6] + assert metadata["ensemble_member_color_deltas"] == [0.02, 0.08] + assert metadata["ensemble_member_magnitude_deltas"] == [0.1, 0.4] + assert ensemble_metadata["member_count"] == 2 + assert ensemble_metadata["members"][0]["label"] == "C1" + assert ensemble_metadata["members"][0]["ra_deg"] == pytest.approx(10.1) + assert ensemble_metadata["members"][0]["dec_deg"] == pytest.approx(-20.1) + assert ensemble_metadata["members"][1]["label"] == "C2" + assert ensemble_metadata["members"][1]["ra_deg"] == pytest.approx(10.2) + assert ensemble_metadata["members"][1]["dec_deg"] == pytest.approx(-20.2) + assert "Target,2450000.12345,12.3400,0.0200,V,NO,STD,ENSEMBLE (2 stars),na" in output_text + assert output_text.rstrip().endswith(",na,1.2346,0.0079") + + +def test_aid_output_samples_large_derived_anchor_label_lists(tmp_path): + fit = DummyFit() + p_dict = { + "pName": "HAT-P-32 b", + "sName": "HAT-P-32", + } + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "aavso_num": "RTZ", + "camera": "CCD", + "filter": "V", + "lat": "+32.41638889", + "long": "-110.73444444", + "elev": 2616, + } + anchor_labels = [f"NextAstro-{index}" for index in range(20)] + vsp_params = [{ + "time": 2450000.12345, + "mag": 12.34, + "mag_err": 0.05, + "airmass": 1.234, + "cname": "RA=10.1000000 Dec=-20.2000000", + "cmag": 12.1, + "cmag_err": 0.03, + "pos": [493, 202], + "catalog_source": "Derived from full-field catalog-calibrated stars", + "is_aavso_vsp": False, + "derived_catalog_reference": True, + "derived_reference_anchor_count": len(anchor_labels), + "derived_reference_anchor_labels": anchor_labels, + "mag_band": "V", + }] + + AIDOutputFiles(fit, p_dict, i_dict, auid=None, chart_id=None, vsp_params=vsp_params).aavso() + + output_text = ( + tmp_path / "AAVSO_Files" / "AID_AAVSO_HAT-P-32_2020-01-01.txt" + ).read_text(encoding="utf-8") + metadata = aavso_json_header(output_text, "COMPARISON-CATALOG-XC") + + assert metadata["derived_reference_anchor_count"] == 20 + assert "derived_reference_anchor_labels" not in metadata + assert metadata["derived_reference_anchor_label_sample"] == anchor_labels[:10] + + +def test_aid_output_floors_reported_magnitude_errors(tmp_path): + fit = DummyFit() + p_dict = { + "pName": "HAT-P-32 b", + "sName": "HAT-P-32", + } + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "aavso_num": "RTZ", + "camera": "CCD", + "filter": "V", + "lat": "+32.41638889", + "long": "-110.73444444", + "elev": 2616, + } + vsp_params = [{ + "time": 2450000.12345, + "mag": 12.34, + "mag_err": 0.0, + "airmass": 1.234, + "cname": "RA=10.1000000 Dec=-20.2000000", + "cmag": 12.1, + "cmag_err": 0.0, + "pos": [493, 202], + "catalog_source": "NextAstro photometry catalog", + "is_aavso_vsp": False, + "mag_band": "V", + }] + + AIDOutputFiles(fit, p_dict, i_dict, auid=None, chart_id=None, vsp_params=vsp_params).aavso() + + output_text = ( + tmp_path / "AAVSO_Files" / "AID_AAVSO_HAT-P-32_2020-01-01.txt" + ).read_text(encoding="utf-8") + metadata = aavso_json_header(output_text, "COMPARISON-CATALOG-XC") + + assert metadata["apparent_magnitude_error"] == pytest.approx(0.001) + assert "HAT-P-32,2450000.12345,12.3400,0.0010,V,NO,STD" in output_text + + +def test_aid_output_skips_over_30_magnitude_rows(tmp_path): + fit = DummyFit() + p_dict = { + "pName": "HAT-P-32 b", + "sName": "HAT-P-32", + } + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "aavso_num": "RTZ", + "camera": "CCD", + "filter": "V", + "lat": "+32.41638889", + "long": "-110.73444444", + "elev": 2616, + } + vsp_params = [{ + "time": 2450000.12345, + "mag": 99.99, + "mag_err": 0.05, + "airmass": 1.234, + "cname": "Comp", + "cmag": 12.1, + "cmag_err": 0.03, + "pos": [493, 202], + }] + + AIDOutputFiles(fit, p_dict, i_dict, auid=None, chart_id=None, vsp_params=vsp_params).aavso() + + output_text = ( + tmp_path / "AAVSO_Files" / "AID_AAVSO_HAT-P-32_2020-01-01.txt" + ).read_text(encoding="utf-8") + + assert "HAT-P-32,2450000.12345" not in output_text + + +def test_save_comp_star_calibration_summary_writes_selected_star(tmp_path): + summary_path = save_comp_star_calibration_summary( + tmp_path, + "HAT-P-32 b", + "2026-03-09", + "PSF photometry", + 0.0012, + [ + { + "label": "Comp 1", + "position": [101, 202], + "selected": True, + "aggregate_score": 0.0012, + "ensemble_score": 0.0010, + "pairwise_median_score": 0.0011, + "pairwise_max_score": 0.0014, + "self_score": 0.0009, + "valid_pair_count": 2, + "coverage_rejected": False, + "suitability_outlier_rejected": False, + }, + { + "label": "Comp 2", + "position": [303, 404], + "selected": False, + "aggregate_score": 0.0031, + "ensemble_score": 0.0028, + "pairwise_median_score": 0.0030, + "pairwise_max_score": 0.0035, + "self_score": 0.0012, + "valid_pair_count": 2, + "coverage_rejected": False, + "suitability_outlier_rejected": True, + }, + ], + 0, + ) + + text = summary_path.read_text() + assert "# Selected comparison star,1" in text + assert "intercomparison_score" in text + assert "intercomparison_frame_rejected_count" in text + assert "ensemble_score" not in text + assert "suitability_outlier_rejected" in text + assert "overexposure_rejected_count" in text + assert "Comp 1,101,202,true" in text + + +def test_final_planetary_params_reports_skipped_airmass_correction(tmp_path): + fit = DummyFit() + fit.airmass_fit_skipped = True + fit.airmass_correction_note = "Skipped (airmass span 0.0400 <= 0.05); no airmass correction applied." + (tmp_path / "working_artifacts").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=False, + vsp_params=[], + ) + + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" + output_text = output_file.read_text(encoding="utf-8") + + assert "Airmass correction" in output_text + assert "no airmass correction applied" in output_text + assert "Airmass coefficient 1 (a1)" not in output_text + + +def test_final_planetary_params_reports_nextastro_variability_reference(tmp_path): + fit = DummyFit() + (tmp_path / "working_artifacts").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + vsp_params = [{ + "cname": "RA=10.1000000 Dec=-20.2000000", + "cmag": 12.345, + "cmag_err": 0.067, + "pos": [493, 202], + "comp_ra": 10.1, + "comp_dec": -20.2, + "catalog_source": "NextAstro photometry catalog", + "is_aavso_vsp": False, + "mag_band": "V", + }] + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=False, + vsp_params=vsp_params, + ) + + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" + final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] + + reference = final_params["Variable Reference Star"] + assert "NextAstro photometry catalog" in reference + assert "RA=10.1000000" in reference + assert "Dec=-20.2000000" in reference + assert "V=12.3450 +/- 0.0670" in reference + + +def test_transit_outputs_use_rprs_fallback_uncertainty_when_model_error_missing(tmp_path): + fit = DummyFit() + fit.errors.pop("rprs") + fit.rprs_prior_fallback_applied = True + fit.rprs_prior_fallback_data_uncertainty = 0.005 + fit.rprs_prior_fallback_note = "Rp/R* fixed to prior." + (tmp_path / "working_artifacts").mkdir() + + p_dict = { + "pName": "HAT-P-32 b", + "pPer": 2.15, + "pPerUnc": 0.001, + "rprs": 0.1, + "rprsUnc": 0.001, + "aRs": 12.0, + "aRsUnc": 0.4, + "inc": 88.5, + "incUnc": 0.2, + "ecc": 0.0, + } + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "filter": "V", + "filter_desc": "Johnson V", + "wl_min": None, + "wl_max": None, + } + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=False, + vsp_params=[], + ) + final_params = json.loads( + (tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json").read_text(encoding="utf-8") + )["FINAL PLANETARY PARAMETERS"] + + assert "0.005" in final_params["Ratio of Planet to Stellar Radius (Rp/R*)"] + + _, _, results = aavso_dicts( + p_dict, + fit, + i_dict, + [0.1], + (0.1, 0.01), + (0.2, 0.02), + (0.3, 0.03), + (0.4, 0.04), + ) + + assert results["Rp/R*"]["uncertainty"] == "0.0050" + + +def test_final_planetary_params_reports_transit_comparison_catalog_reference(tmp_path): + fit = DummyFit() + (tmp_path / "working_artifacts").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + vsp_params = [ + { + "cname": "000-BJX-718", + "cmag": 9.751, + "cmag_err": 0.018, + "pos": [616, 113], + "catalog_source": "AAVSO VSP", + "is_aavso_vsp": True, + "mag_band": "V", + }, + { + "cname": "000-BJX-718", + "cmag": 9.751, + "cmag_err": 0.018, + "pos": [616, 113], + "catalog_source": "AAVSO VSP", + "is_aavso_vsp": True, + "mag_band": "V", + }, + ] + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=True, + vsp_params=vsp_params, + comp_star=1, + comp_coords=[616, 113], + min_aper=2.7, + min_annul=10.15, + ) + + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" + final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] + + assert final_params["Transit Fit Comparison Star"] == "#1 - [616, 113]" + assert "Best Comparison Star" not in final_params + assert final_params["Variable Reference Star"] == "AAVSO Label: 000-BJX-718, Position: [616, 113]" + assert "Remeasured 2 out-of-transit target/reference point(s)" in final_params["Variable Reference Measurement"] + assert "AID rows list the BJD_TDB timestamps used" in final_params["Variable Reference Measurement"] + assert "transit-fit catalog reference" in final_params["Variable Reference Measurement"] + + +def test_final_planetary_params_suppresses_variable_reference_without_transit_comparison(tmp_path): + fit = DummyFit() + (tmp_path / "working_artifacts").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + vsp_params = [{ + "cname": "000-BJX-718", + "cmag": 9.751, + "cmag_err": 0.018, + "pos": [616, 113], + "catalog_source": "AAVSO VSP", + "is_aavso_vsp": True, + "mag_band": "V", + }] + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=True, + vsp_params=vsp_params, + comp_star=None, + comp_coords=None, + min_aper=-2.7, + min_annul=10.15, + ) + + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" + final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] + + assert final_params["Transit Fit Comparison Star"] == "None" + assert "Variable Reference Star" not in final_params + assert "Variable Reference Measurement" not in final_params + + +def test_final_planetary_params_reports_ars_and_impact_parameter_under_inclination(tmp_path): + fit = DummyFit() + (tmp_path / "working_artifacts").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=False, + vsp_params=[], + ) + + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" + output_data = json.loads(output_file.read_text(encoding="utf-8")) + final_params = output_data["FINAL PLANETARY PARAMETERS"] + keys = list(final_params) + inclination_index = keys.index("Orbital Inclination (inc)") + + assert keys[inclination_index + 1] == "Ratio of Distance to Stellar Radius (a/Rs)" + assert keys[inclination_index + 2] == "Impact Parameter (b)" + assert final_params["Ratio of Distance to Stellar Radius (a/Rs)"] == "12.00 +/- 0.40" + + expected_b, expected_b_error = fit_impact_parameter_value_error(fit) + assert expected_b == pytest.approx(12.0 * np.cos(np.deg2rad(88.5))) + assert final_params["Impact Parameter (b)"] == "0.314 +/- 0.043" + + +def test_final_planetary_params_matches_values_to_two_sigfig_uncertainties(tmp_path): + fit = DummyFit() + fit.errors["a1"] = 0.00023 + fit.errors["a2"] = 0.0031 + (tmp_path / "working_artifacts").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "filter": "V", + "filter_desc": "Johnson V", + "wl_min": None, + "wl_max": None, + } + + OutputFiles(fit, p_dict, i_dict, [0.063, 0.083]).final_planetary_params( + phot_opt=False, + vsp_params=[], + ) + + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" + final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] + + assert final_params["Flux normalization (a1)"] == "1.00000 +/- 0.00023" + assert final_params["Airmass coefficient 2 (a2)"] == "0.0000 +/- 0.0031" + assert final_params["Transit Duration (day)"] == "0.073 +/- 0.010" + + +def test_detrended_fixed_baseline_does_not_report_inherited_errors_as_fitted(tmp_path): + fit = DummyFit() + fit.errors["a1"] = 0.00023 + fit.errors["a2"] = 0.0031 + fit.oot_baseline_detrending_applied = True + fit.pre_detrending_baseline_source = "test out-of-transit baseline fit" + fit.pre_detrending_baseline_scale_parameter = "a1" + fit.pre_detrending_baseline_scale_value = 1.004321 + fit.pre_detrending_baseline_scale_error = 0.00023 + fit.pre_detrending_baseline_a2_value = -0.01234 + fit.pre_detrending_baseline_a2_error = 0.0031 + (tmp_path / "working_artifacts").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "filter": "V", + "filter_desc": "Johnson V", + "wl_min": None, + "wl_max": None, + } + + OutputFiles(fit, p_dict, i_dict, [0.063, 0.083]).final_planetary_params( + phot_opt=False, + vsp_params=[], + ) + + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" + final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] + _, _, results = aavso_dicts( + { + **p_dict, + "pPer": 2.15, + "pPerUnc": 0.001, + "rprs": 0.1, + "rprsUnc": 0.001, + "aRs": 12.0, + "aRsUnc": 0.4, + "inc": 88.5, + "incUnc": 0.2, + "ecc": 0.0, + }, + fit, + i_dict, + [0.063, 0.083], + (0.1, 0.01), + (0.2, 0.02), + (0.3, 0.03), + (0.4, 0.04), + ) + + assert final_params["Flux normalization (a1)"] == ( + "1.0 (fixed after out-of-transit baseline detrending)" + ) + assert final_params["Airmass coefficient 2 (a2)"] == ( + "0.0 (fixed after out-of-transit baseline detrending)" + ) + assert final_params["Pre-detrending baseline source"] == "test out-of-transit baseline fit" + assert final_params["Pre-detrending airmass coefficient 1 (a1)"] == ( + "1.00432 +/- 0.00023" + ) + assert final_params["Pre-detrending airmass coefficient 2 (a2)"] == ( + "-0.0123 +/- 0.0031" + ) + assert results["Am1"] == {"value": "1.0", "uncertainty": "0"} + assert results["Am2"] == {"value": "0.0", "uncertainty": "0"} + + +def test_final_planetary_params_reports_fit_uncertainties_not_prior_uncertainties(tmp_path): + fit = DummyFit() + (tmp_path / "working_artifacts").mkdir() + + p_dict = { + "pName": "HAT-P-32 b", + "midTUnc": 9.9, + "rprsUnc": 8.8, + "aRsUnc": 7.7, + "incUnc": 6.6, + } + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=False, + vsp_params=[], + ) + + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" + final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] + + assert final_params["Mid-Transit Time (Tmid)"].endswith("+/- 0.00010 BJD_TDB") + assert final_params["Ratio of Planet to Stellar Radius (Rp/R*)"] == "0.1234 +/- 0.0010" + assert "Transit depth (Rp/Rs)^2" not in final_params + assert AREA_DEPTH_LABEL in final_params + assert OBSERVABLE_DEPTH_LABEL in final_params + assert PRIOR_OBSERVABLE_DEPTH_LABEL in final_params + assert OBSERVABLE_DEPTH_DELTA_LABEL in final_params + assert final_params["Orbital Inclination (inc)"] == "88.50 +/- 0.20 " + assert final_params["Ratio of Distance to Stellar Radius (a/Rs)"] == "12.00 +/- 0.40" + assert final_params["Impact Parameter (b)"] == "0.314 +/- 0.043" + + +def test_fit_empirical_transit_uncertainty_uses_residual_scatter_and_point_counts(): + fit = DummyFit() + fit.parameters["rprs"] = 0.1 + fit.errors["rprs"] = 0.002 + fit.transit = np.array([1.0, 1.0, 0.99, 0.99, 1.0, 1.0]) + fit.model = np.array(fit.transit) + fit.data = fit.model + np.array([0.0, 0.01, -0.01, 0.01, -0.01, 0.0]) + fit.residuals = fit.data - fit.model + fit.dataerr = np.full_like(fit.model, 0.01) + fit.airmass_model = np.ones_like(fit.model) + + empirical = fit_empirical_transit_uncertainty(fit) + + assert empirical["available"] is True + assert empirical["in_transit_point_count"] == 2 + assert empirical["out_of_transit_point_count"] == 4 + assert empirical["data_rprs_uncertainty"] == pytest.approx( + empirical["depth_uncertainty_fraction"] / 0.2 + ) + assert empirical["depth_flux_scatter_fraction"] == pytest.approx( + empirical["residual_scatter"] + ) + assert empirical["data_rprs_standard_error"] == pytest.approx( + empirical["depth_standard_error_fraction"] / 0.2 + ) + assert empirical["data_rprs_flux_scatter_uncertainty"] == pytest.approx( + empirical["depth_flux_scatter_fraction"] / 0.2 + ) + assert empirical["red_noise_beta_factor"] >= 1.0 + assert empirical["data_rprs_uncertainty"] >= empirical["data_rprs_standard_error"] + assert empirical["data_rprs_flux_scatter_uncertainty"] > empirical["data_rprs_standard_error"] + assert empirical["combined_rprs_uncertainty"] > empirical["model_rprs_uncertainty"] + assert empirical["baseline_red_noise_uncertainty_fraction"] >= ( + empirical["baseline_standard_error_fraction"] + ) + assert empirical["depth_uncertainty_fraction"] >= empirical["baseline_red_noise_uncertainty_fraction"] + + +def test_fit_empirical_transit_uncertainty_uses_data_only_for_prior_fallback(): + fit = DummyFit() + fit.parameters["rprs"] = 0.1 + fit.errors["rprs"] = 0.5 + fit.rprs_prior_fallback_applied = True + fit.rprs_prior_fallback_note = "Applied Rp/R* prior fallback." + fit.transit = np.array([1.0, 1.0, 0.99, 0.99, 1.0, 1.0]) + fit.model = np.array(fit.transit) + fit.data = fit.model + np.array([0.0, 0.01, -0.01, 0.01, -0.01, 0.0]) + fit.residuals = fit.data - fit.model + fit.dataerr = np.full_like(fit.model, 0.01) + fit.airmass_model = np.ones_like(fit.model) + + empirical = fit_empirical_transit_uncertainty(fit) + + assert empirical["rprs_uncertainty_basis"] == "prior_assumed_data_only" + assert np.isnan(empirical["model_rprs_uncertainty"]) + assert empirical["combined_rprs_uncertainty"] == pytest.approx( + empirical["data_rprs_uncertainty"] + ) + assert empirical["conservative_rprs_uncertainty"] == pytest.approx( + empirical["data_rprs_uncertainty"] + ) + + +def test_final_planetary_params_reports_model_and_red_noise_uncertainties(tmp_path): + fit = DummyFit() + fit.parameters["rprs"] = 0.1 + fit.errors["rprs"] = 0.002 + fit.transit = np.array([1.0, 1.0, 0.99, 0.99, 1.0, 1.0]) + fit.model = np.array(fit.transit) + fit.data = fit.model + np.array([0.0, 0.01, -0.01, 0.01, -0.01, 0.0]) + fit.residuals = fit.data - fit.model + fit.dataerr = np.full_like(fit.model, 0.01) + fit.airmass_model = np.ones_like(fit.model) + (tmp_path / "working_artifacts").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=False, + vsp_params=[], + ) + + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" + final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] + + assert final_params["Ratio of Planet to Stellar Radius (Rp/R*)"] == ( + final_params["Ratio of Planet to Stellar Radius (Rp/R*) model+red-noise uncertainty"] + ) + assert final_params["Ratio of Planet to Stellar Radius (Rp/R*) model-fit uncertainty"] == ( + "0.1000 +/- 0.0020" + ) + assert "Ratio of Planet to Stellar Radius (Rp/R*) data-fit red-noise uncertainty" in final_params + assert "Ratio of Planet to Stellar Radius (Rp/R*) model+red-noise uncertainty" in final_params + assert "Ratio of Planet to Stellar Radius (Rp/R*) data-fit standard-error estimate" in final_params + assert "Ratio of Planet to Stellar Radius (Rp/R*) flux-scatter equivalent" in final_params + assert "Transit depth red-noise uncertainty" in final_params + assert "Transit depth data-fit standard-error estimate" in final_params + assert "Transit depth flux-scatter equivalent" in final_params + assert final_params[AREA_DEPTH_LABEL] == ( + final_params[f"{AREA_DEPTH_LABEL} model+red-noise uncertainty"] + ) + assert final_params["Mid-Transit Time (Tmid)"] == ( + final_params["Mid-Transit Time (Tmid) model+red-noise uncertainty"] + ) + assert "Mid-Transit Time (Tmid) model-fit uncertainty" in final_params + assert final_params["Orbital Inclination (inc)"] == ( + final_params["Orbital Inclination (inc) model+red-noise uncertainty"] + ) + assert "Orbital Inclination (inc) model-fit uncertainty" in final_params + assert final_params["Ratio of Distance to Stellar Radius (a/Rs)"] == ( + final_params["Ratio of Distance to Stellar Radius (a/Rs) model+red-noise uncertainty"] + ) + assert "Ratio of Distance to Stellar Radius (a/Rs) model-fit uncertainty" in final_params + assert final_params["Impact Parameter (b)"] == ( + final_params["Impact Parameter (b) model+red-noise uncertainty"] + ) + assert "Impact Parameter (b) model-fit uncertainty" in final_params + assert f"{AREA_DEPTH_LABEL} model-fit uncertainty" in final_params + assert f"{AREA_DEPTH_LABEL} data-fit red-noise uncertainty" in final_params + assert "Flux baseline red-noise uncertainty" in final_params + assert "Flux baseline standard-error estimate" in final_params + assert "Red-noise beta factor" in final_params + assert final_params["Data-fit uncertainty point counts"] == "2 in transit, 4 out of transit" + assert "primary Rp/R*" in final_params["Uncertainty interpretation note"] + assert "baseline component" in final_params["Uncertainty interpretation note"] + assert "time-binning" in final_params["Uncertainty interpretation note"] + + +def test_final_planetary_params_reports_prior_fallback_data_only_uncertainty(tmp_path): + fit = DummyFit() + fit.parameters["rprs"] = 0.1 + fit.errors["rprs"] = 0.5 + fit.rprs_prior_fallback_applied = True + fit.rprs_prior_fallback_prior_value = 0.1 + fit.rprs_prior_fallback_original_fit_value = 0.11 + fit.rprs_prior_fallback_data_uncertainty = 0.02 + fit.rprs_prior_fallback_note = "Applied Rp/R* prior fallback." + fit.transit = np.array([1.0, 1.0, 0.99, 0.99, 1.0, 1.0]) + fit.model = np.array(fit.transit) + fit.data = fit.model + np.array([0.0, 0.01, -0.01, 0.01, -0.01, 0.0]) + fit.residuals = fit.data - fit.model + fit.dataerr = np.full_like(fit.model, 0.01) + fit.airmass_model = np.ones_like(fit.model) + (tmp_path / "working_artifacts").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=False, + vsp_params=[], + ) + + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" + final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] + + assert final_params["Rp/R* uncertainty basis"] == "prior_assumed_data_only" + assert not any("Rp/R*) model-fit uncertainty" in key for key in final_params) + assert not any("Rp/R*) model+standard-error" in key for key in final_params) + assert "input prior Rp/R* value with a data-only" in final_params["Uncertainty interpretation note"] + assert "Rp/R* prior fallback note" in final_params + assert any("prior-assumed data-only uncertainty" in key for key in final_params) + + +def test_final_planetary_params_can_publish_accepted_copy_to_root(tmp_path): + fit = DummyFit() + (tmp_path / "working_artifacts").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=False, + vsp_params=[], + publish_to_root=True, + ) + + temp_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" + root_file = tmp_path / "FinalParams_HAT-P-32b_2020-01-01.json" + + assert temp_file.exists() + assert root_file.exists() + assert root_file.read_text(encoding="utf-8") == temp_file.read_text(encoding="utf-8") + + +def test_final_planetary_params_reports_adaptive_aperture_summary(tmp_path): + fit = DummyFit() + (tmp_path / "working_artifacts").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + adaptive_summary = { + "aperture_sigma": 2.62, + "annulus_sigma": 9.00, + "aperture_median": 7.98, + "aperture_std": 0.41, + "aperture_min": 7.12, + "aperture_max": 8.76, + "annulus_median": 27.43, + "annulus_std": 1.39, + "annulus_min": 25.11, + "annulus_max": 30.08, + } + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=True, + vsp_params=[], + comp_star=9, + comp_coords=[1446.0, 2399.0], + min_aper=7.98, + min_annul=27.43, + adaptive_summary=adaptive_summary, + ) + + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" + output_text = output_file.read_text(encoding="utf-8") + + assert "Adaptive Aperture Scale" in output_text + assert "2.62 sigma" in output_text + assert "Optimal Aperture" in output_text + assert "7.98 +/- 0.41 px" in output_text + assert "Aperture Range" in output_text + assert "7.12 to 8.76 px" in output_text + + +def test_final_planetary_params_reports_transit_qc_summary(tmp_path): + fit = DummyFit() + fit.transit_qc = { + "status": "pass", + "summary": "Transit model strongly preferred over flat/null model (Delta BIC=18.40, Delta chi2=27.10).", + "delta_bic": 18.4, + "delta_chi2": 27.1, + "rprs_sigma": 6.2, + "duration_ratio": 1.05, + "eebls_depth_snr": 5.8, + "residual_scatter": 0.0032, + "deviation_from_expected_value": 0.91, + "tmid_deviation_sigma": 1.1, + "tmid_deviation_minutes": 3.2, + "tmid_deviation_threshold_minutes": 14.4, + "expected_tmid_unc_minutes": 2.88, + "rprs_deviation_fit_unc": 0.0046, + "rprs_deviation_sigma": 0.8, + "deviation_sigma_threshold": 5.0, + "ktmf_metric": 4.63, + "ktmf_contributions": [ + { + "label": "Delta BIC", + "available": True, + "points": 1.25, + "max_points": 1.40, + "score": 0.89, + "detail": "Delta BIC=18.40", + }, + { + "label": "Deviation From Expected Value", + "available": True, + "points": 0.91, + "max_points": 1.00, + "score": 0.91, + "detail": "score=0.91, Rp/R* sigma=0.80, fit uncertainty=0.004600", + }, + ], + "notes": ["The transit model is strongly preferred over the flat/null model."], + } + (tmp_path / "working_artifacts").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=False, + vsp_params=[], + ) + + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" + output_text = output_file.read_text(encoding="utf-8") + + assert "Transit detection QC" in output_text + assert "Transit vs flat model" in output_text + assert "PASS" in output_text + assert "Delta BIC=18.40" in output_text + assert "Residual scatter around full model fit" in output_text + assert "Deviation From Expected Value" in output_text + assert "3.20 minutes" not in output_text + assert "Expected-value Tmid offset" not in output_text + assert "Expected-value Tmid QC window" not in output_text + assert "KTMF" in output_text + assert "KTMF contribution 1" in output_text + + +def test_final_planetary_params_reports_ktmf_decision_details(tmp_path): + fit = DummyFit() + fit.transit_qc = { + "status": "pass", + "summary": "Transit model strongly preferred over flat/null model.", + "ktmf_metric": 4.63, + "delta_bic": 18.4, + "delta_chi2": 27.1, + "tmid_gaussianity_score": 0.94, + "tmid_gaussianity_score_uncertainty": 0.03, + "tmid_gaussianity_effective_sample_count": 1840.0, + "tmid_gaussianity_detail": "strongly Gaussian-like", + "ktmf_contributions": [ + { + "label": "EEBLS Depth SNR", + "available": True, + "points": 0.74, + "max_points": 0.80, + "score": 0.93, + "detail": "5.80", + }, + { + "label": "Tmid Posterior Gaussianity", + "available": True, + "points": 0.94, + "max_points": 1.00, + "score": 0.94, + "score_uncertainty": 0.03, + "detail": "strongly Gaussian-like", + }, + ], + } + (tmp_path / "working_artifacts").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + photometry_info = { + "selection_basis": "comparison_field_retry", + "selection_metric": "ktmf", + "comp_star_num": 2, + "comparison_ktmf_metric": 4.60, + "comparison_eebls_snr": 5.2, + "comparison_transit_delta_bic": 18.4, + "selected_comparison_selection_reason": "selected: highest KTMF among candidates", + "selected_comparison_ktmf_contributions": [ + { + "label": "Residual Scatter Around Full Model Fit", + "available": True, + "points": 0.63, + "max_points": 0.70, + "score": 0.90, + "detail": "0.3500%", + } + ], + "comparison_fit_attempt_summaries": [ + { + "rank": 1, + "comp_index": 0, + "label": "Comp 1", + "selected": False, + "selection_reason": "not selected: KTMF 3.20/5.00 was lower than the selected 4.60/5.00", + "ktmf_metric": 3.2, + "transit_delta_bic": 8.1, + "eebls_snr": 4.2, + "transit_qc_status": "marginal", + }, + { + "rank": 2, + "comp_index": 1, + "label": "Comp 2", + "selected": True, + "selection_reason": "selected: highest KTMF among candidates", + "ktmf_metric": 4.6, + "transit_delta_bic": 18.4, + "eebls_snr": 5.2, + "transit_qc_status": "pass", + }, + ], + } + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=True, + vsp_params=[], + comp_star=2, + comp_coords=[300.5, 400.5], + min_aper=7.5, + min_annul=22.5, + photometry_info=photometry_info, + ) + + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" + output_data = json.loads(output_file.read_text(encoding="utf-8")) + final_params = output_data["FINAL PLANETARY PARAMETERS"] + final_param_keys = list(final_params) + + assert final_param_keys[:2] == ["Transit detection QC", "KTMF"] + assert final_params["Transit detection QC"] == "PASS" + assert final_params["KTMF"] == "4.63 / 5.00" + assert final_params["KTMF target-fit decision"] == "PASS: KTMF=4.63 / 5.00" + assert final_params["KTMF comparison selection mode"] == "basis=comparison_field_retry, metric=ktmf" + assert "selected: highest KTMF" in final_params["KTMF selected comparison decision"] + assert "Comp 1" in final_params["KTMF comparison candidate 1"] + assert "not selected: KTMF" in final_params["KTMF comparison candidate 1"] + assert "Residual Scatter Around Full Model Fit" in final_params["KTMF selected comparison contribution 1"] + assert "Tmid Posterior Gaussianity" in final_params["KTMF target contribution 2"] + + qc_metadata = build_aavso_qc_metadata(fit) + assert qc_metadata["tmid_gaussianity_score"] == pytest.approx(0.94) + assert qc_metadata["tmid_gaussianity_score_uncertainty"] == pytest.approx(0.03) + assert qc_metadata["tmid_gaussianity_effective_sample_count"] == pytest.approx(1840.0) + + +def test_final_planetary_params_reports_absolute_fit_quality(tmp_path): + fit = DummyFit() + fit.data = np.array([1.0, 1.02, 0.98, 1.01, 0.99, 1.0]) + fit.model = np.ones(6, dtype=float) + fit.residuals = fit.data - fit.model + fit.dataerr = np.full(6, 0.01, dtype=float) + fit.time = np.arange(6, dtype=float) + fit.airmass_model = np.ones(6, dtype=float) + fit.bounds = {"tmid": [0, 1], "rprs": [0, 1], "a1": [0, 2]} + (tmp_path / "working_artifacts").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=False, + vsp_params=[], + ) + + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" + output_data = json.loads(output_file.read_text(encoding="utf-8")) + final_params = output_data["FINAL PLANETARY PARAMETERS"] + + assert final_params["Fit quality reduced chi-square"] == "3.333" + assert final_params["Fit quality chi-square"] == "10.00" + assert final_params["Fit quality degrees of freedom"] == "3" + assert final_params["Fit quality RMS residual"] == "1.2910 %" + assert final_params["Fit quality median absolute normalized residual"] == "1.00 sigma" + assert final_params["Fit quality RMS residual / median uncertainty"] == "1.29" + assert final_params["Fit quality point count"] == "6" + + +def test_final_planetary_params_reports_prior_assumed_geometry_note(tmp_path): + fit = DummyFit() + fit.partial_transit_geometry_prior_assumption_note = ( + "Applied prior-assumed transit geometry for a one-sided partial light curve." + ) + (tmp_path / "working_artifacts").mkdir() + + p_dict = {"pName": "HAT-P-32 b"} + i_dict = {"save": str(tmp_path), "date": "2020-01-01"} + + OutputFiles(fit, p_dict, i_dict, [0.1]).final_planetary_params( + phot_opt=False, + vsp_params=[], + ) + + output_file = tmp_path / "working_artifacts" / "FinalParams_HAT-P-32b_2020-01-01.json" + final_params = json.loads(output_file.read_text(encoding="utf-8"))["FINAL PLANETARY PARAMETERS"] + + assert ( + final_params["Prior-assumed partial-transit geometry note"] + == "Applied prior-assumed transit geometry for a one-sided partial light curve." + ) + + +def test_aavso_output_writes_zero_airmass_terms_when_correction_is_skipped(tmp_path): + fit = DummyFit() + fit.airmass_fit_skipped = True + fit.airmass_correction_note = "Skipped (input AAVSO file already reports AIRMASS, AIRMASS CORRECTION FUNCTION); no airmass correction applied." + + p_dict = { + "pName": "HAT-P-32 b", + "sName": "HAT-P-32", + "pPer": 2.1500082, + "pPerUnc": 1.3e-07, + "rprs": 0.1488623525, + "rprsUnc": 0.0005539487, + "aRs": 5.344, + "aRsUnc": 0.03949, + "inc": 88.98, + "incUnc": 0.7602, + "ecc": 0.159, + "dist": None, + "pm_ra": None, + "pm_dec": None, + } + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "aavso_num": "RTZ", + "second_obs": "", + "obs_name": "", + "camera": "CCD", + "pixel_bin": "1x1", + "exposure": 60.0, + "lat": "+32.41638889", + "long": "-110.73444444", + "elev": 2616, + "notes": "na", + "filter": "CV", + "filter_desc": "Clear with V zero-point", + "wl_min": None, + "wl_max": None, + } + + OutputFiles(fit, p_dict, i_dict, [0.1]).aavso( + {"ra": "", "dec": "", "x": "493", "y": "202"}, + [1.0], + (0.1, 0.01), + (0.2, 0.01), + (0.3, 0.01), + (0.4, 0.01), + None, + ) + + output_file = tmp_path / "AAVSO_Files" / "AAVSO_HAT-P-32b_2020-01-01.txt" + output_text = output_file.read_text(encoding="utf-8") + + assert "Am1=0 +/- 0" in output_text + assert "Am2=0 +/- 0" in output_text + assert output_text.strip().endswith("1.0") + + +def test_aavso_output_includes_extended_diagnostic_comment_headers(tmp_path): + fit = DummyFit() + fit.data = np.array([1.0, 1.02, 0.98, 1.01, 0.99, 1.0]) + fit.model = np.ones(6, dtype=float) + fit.residuals = fit.data - fit.model + fit.dataerr = np.full(6, 0.01, dtype=float) + fit.time = np.arange(6, dtype=float) + 2450000.0 + fit.airmass_model = np.ones(6, dtype=float) + fit.bounds = {"tmid": [0, 1], "rprs": [0, 1], "a1": [0, 2]} + fit.transit_qc = { + "computed": True, + "status": "pass", + "summary": "Transit model strongly preferred over flat/null model.", + "delta_bic": 18.4, + "delta_chi2": 27.1, + "residual_scatter": 0.0032, + "rprs_sigma": 6.2, + "duration_ratio": 1.05, + "eebls_depth_snr": 5.8, + "deviation_from_expected_value": 0.91, + "tmid_deviation_minutes": 3.2, + "tmid_deviation_sigma": 1.1, + "rprs_deviation_sigma": 0.8, + "ktmf_metric": 4.63, + "ktmf_contributions": [ + { + "label": "EEBLS Depth SNR", + "available": True, + "points": 0.74, + "max_points": 0.80, + "score": 0.93, + "detail": "5.80", + } + ], + } + fit.frame_filter_diagnostics = [ + { + "stage": "Final-fit phase residual clip", + "input_point_count": 4, + "kept_point_count": 3, + "dropped_point_count": 1, + "dropped_ranges": [{"start": 2450000.2, "end": 2450000.2, "count": 1}], + } + ] + p_dict = { + "pName": "HAT-P-32 b", + "sName": "HAT-P-32", + "pPer": 2.1500082, + "pPerUnc": 1.3e-07, + "rprs": 0.1488623525, + "rprsUnc": 0.0005539487, + "aRs": 5.344, + "aRsUnc": 0.03949, + "inc": 88.98, + "incUnc": 0.7602, + "ecc": 0.159, + "dist": 245.7, + "pm_ra": 14.25, + "pm_dec": -9.5, + } + i_dict = { + "save": str(tmp_path), + "date": "2020-01-01", + "aavso_num": "RTZ", + "second_obs": "", + "obs_name": "", + "camera": "CCD", + "pixel_bin": "1x1", + "exposure": 60.0, + "lat": "+32.41638889", + "long": "-110.73444444", + "elev": 2616, + "notes": "na", + "filter": "CV", + "filter_desc": "Clear with V zero-point", + "wl_min": None, + "wl_max": None, + } + photometry_info = { + "comp_star_num": 2, + "comp_star_coords": [300.5, 400.5], + "min_aperture": 7.5, + "min_annulus": 22.5, + "aperture_index": 1, + "annulus_index": 2, + "calibration_field_score": 0.0042, + "selection_basis": "comparison_field", + "selection_metric": "ktmf", + "comparison_ktmf_metric": 4.6, + "comparison_eebls_snr": 5.2, + "comparison_transit_delta_bic": 18.4, + "selected_comparison_selection_reason": "selected: highest KTMF among candidates", + "selected_comparison_ktmf_contributions": [ + { + "label": "Residual Scatter Around Full Model Fit", + "available": True, + "points": 0.63, + "max_points": 0.70, + "score": 0.90, + "detail": "0.3500%", + } + ], + "comparison_fit_attempt_summaries": [ + { + "rank": 1, + "comp_index": 0, + "label": "Comp 1", + "selected": False, + "selection_reason": "not selected: KTMF 3.20/5.00 was lower than the selected 4.60/5.00", + "ktmf_metric": 3.2, + "transit_delta_bic": 8.1, + "eebls_snr": 4.2, + "transit_qc_status": "marginal", + "ktmf_contributions": [], + }, + { + "rank": 2, + "comp_index": 1, + "label": "Comp 2", + "selected": True, + "selection_reason": "selected: highest KTMF among candidates", + "ktmf_metric": 4.6, + "transit_delta_bic": 18.4, + "eebls_snr": 5.2, + "transit_qc_status": "pass", + "ktmf_contributions": [ + { + "label": "Residual Scatter Around Full Model Fit", + "available": True, + "points": 0.63, + "max_points": 0.70, + "score": 0.90, + "detail": "0.3500%", + } + ], + }, + ], + "reuse_selected_full_reduction_fit": True, + "selected_source_indices": np.array([0, 2, 3]), + "selected_fit_good_times": np.array([2450000.0, 2450000.1, 2450000.2]), + "adaptive_summary": { + "aperture_sigma": 2.62, + "annulus_sigma": 9.00, + "frame_sigma": np.array([2.0, 2.1, 2.2]), + "fwhm_series": np.array([4.7, 4.8, 4.9]), + "sky_inner_series": np.array([12.0, 12.1, 12.2]), + "sky_outer_series": np.array([18.0, 18.1, 18.2]), + "sky_pixel_series": np.array([200.0, 201.0, 202.0]), + "aperture_median": 7.98, + "aperture_std": 0.41, + "aperture_min": 7.12, + "aperture_max": 8.76, + "annulus_median": 27.43, + "annulus_std": 1.39, + "annulus_min": 25.11, + "annulus_max": 30.08, + }, + } + comp_star_header = {"ra": "10.1", "dec": "-20.2", "x": "493", "y": "202"} + + OutputFiles(fit, p_dict, i_dict, [0.1]).aavso( + comp_star_header, + np.ones(6, dtype=float), + (0.1, 0.01), + (0.2, 0.01), + (0.3, 0.01), + (0.4, 0.01), + "abc123", + photometry_info=photometry_info, + frame_filtering_info={ + "initial_frame_count": 5, + "after_missing_wcs_filter_frame_count": 4, + "after_target_wcs_filter_frame_count": 3, + "final_prephotometry_frame_count": 3, + "ignore_header_wcs": False, + "bad_wcs_threshold_percent": 3.0, + "pointing_rejection_sigma": 3.0, + "dropped_missing_wcs_files": [tmp_path / "missing_wcs.fits"], + "dropped_target_wcs_files": [tmp_path / "target_off_frame.fits"], + "dropped_pointing_files": [tmp_path / "bad_pointing.fits"], + }, + astrometry_info={ + "wcs_file": tmp_path / "wcs.fits", + "coordinate_source": "wcs", + "target_ra_dec_deg": [10.0, -20.0], + "comparison_ra_dec_deg": [[10.1, -20.2]], + }, + bad_pixel_info={ + "enabled": True, + "detected": True, + "bad_pixel_count": 3, + "frame_count": 10, + "required_count": 4, + "minimum_fraction": 0.3, + "counts_path": tmp_path / "working_artifacts" / "BadPixelDetectionCounts.fits", + "mask_path": tmp_path / "working_artifacts" / "BadPixelMask.fits", + }, + ) + + output_text = ( + tmp_path / "AAVSO_Files" / "AAVSO_HAT-P-32b_2020-01-01.txt" + ).read_text(encoding="utf-8") + + results = aavso_json_header(output_text, "RESULTS-XC") + assert "a/R*" in results + assert "Impact Parameter (b)" in results + assert "Transit depth (Rp/R*)^2" not in results + assert results[AREA_DEPTH_LABEL]["units"] == "percent" + assert results[OBSERVABLE_DEPTH_LABEL]["units"] == "percent" + assert results[PRIOR_OBSERVABLE_DEPTH_LABEL]["units"] == "percent" + assert results[OBSERVABLE_DEPTH_DELTA_LABEL]["units"] == "percent" + assert results["Residual scatter around full model fit"]["value"] == "0.32" + + qc = aavso_json_header(output_text, "QC-XC") + assert qc["status"] == "pass" + assert qc["ktmf_metric"] == pytest.approx(4.63) + assert qc["ktmf_contributions"][0]["label"] == "EEBLS Depth SNR" + + fit_quality = aavso_json_header(output_text, "FIT_QUALITY-XC") + assert fit_quality["reduced_chi_square"] == pytest.approx(10.0 / 3.0) + assert fit_quality["chi_square"] == pytest.approx(10.0) + assert fit_quality["degrees_of_freedom"] == 3 + assert fit_quality["median_absolute_normalized_residual"] == pytest.approx(1.0) + + ktmf_decision = aavso_json_header(output_text, "KTMF_DECISION-XC") + assert ktmf_decision["target_fit"]["ktmf_metric"] == pytest.approx(4.63) + assert ktmf_decision["comparison_selection"]["basis"] == "comparison_field" + assert ktmf_decision["comparison_selection"]["metric"] == "ktmf" + assert ktmf_decision["comparison_selection"]["selected"]["selection_reason"] == "selected: highest KTMF among candidates" + assert ktmf_decision["comparison_selection"]["candidate_count"] == 2 + assert ktmf_decision["comparison_selection"]["candidates"][0]["selection_reason"].startswith("not selected: KTMF") + + photometry = aavso_json_header(output_text, "PHOTOMETRY-XC") + assert photometry["selected_comparison_star"] == 2 + assert photometry["comparison_field_score_percent"] == pytest.approx(0.42) + assert photometry["reused_selected_full_reduction_fit"] is True + + aperture = aavso_json_header(output_text, "APERTURE-XC") + assert aperture["adaptive"] is True + assert aperture["aperture_sigma"] == pytest.approx(2.62) + assert aperture["fwhm_px"]["median"] == pytest.approx(4.8) + + frame_filtering = aavso_json_header(output_text, "FRAME_FILTERING-XC") + assert frame_filtering["missing_wcs_rejections"]["files"] == ["missing_wcs.fits"] + assert frame_filtering["target_wcs_rejections"]["files"] == ["target_off_frame.fits"] + assert frame_filtering["pointing_rejections"]["files"] == ["bad_pointing.fits"] + assert frame_filtering["lightcurve_dropped_point_count"] == 1 + + astrometry = aavso_json_header(output_text, "ASTROMETRY-XC") + assert astrometry["wcs_file"] == "wcs.fits" + assert astrometry["comparison_star_aavso_header"] == comp_star_header + + bad_pixel = aavso_json_header(output_text, "BAD_PIXEL-XC") + assert bad_pixel["enabled"] is True + assert bad_pixel["bad_pixel_count"] == 3 + assert bad_pixel["counts_path"] == "BadPixelDetectionCounts.fits" diff --git a/tests/test_plate_status.py b/tests/test_plate_status.py new file mode 100644 index 00000000..d8eb6260 --- /dev/null +++ b/tests/test_plate_status.py @@ -0,0 +1,107 @@ +import csv + +from exotic.plate_status import PlateStatus + + +def test_out_of_frame_warning_reports_only_fits_basename(): + messages = [] + status = PlateStatus(lambda message, **kwargs: messages.append(message)) + status.setCurrentFilename('/mnt/data/session/TIC 13510052901-R-20240403-030-063313_out.fits') + + status.outOfFrameWarning(11) + + assert messages[0] == ( + 'Comparison star #11 is beyond the edge of file ' + 'TIC 13510052901-R-20240403-030-063313_out.fits' + ) + assert 'repeated frame-level star warnings are aggregated' in messages[1] + + +def test_out_of_frame_warning_preserves_fits_fz_basename(): + messages = [] + status = PlateStatus(lambda message, **kwargs: messages.append(message)) + status.setCurrentFilename(r'C:\data\session\compressed-frame.fits.fz') + + status.outOfFrameWarning(0) + + assert messages[0] == 'Target star is beyond the edge of file compressed-frame.fits.fz' + + +def test_tracked_vsx_label_is_used_for_every_star_warning_type(): + messages = [] + status = PlateStatus(lambda message, **kwargs: messages.append(message)) + status.setComparisonStarLabels({18: 'Tracked VSX variable DI Her'}) + + status.setCurrentFilename('/mnt/data/frame-1.fits.fz') + status.outOfFrameWarning(18) + status.setCurrentFilename('/mnt/data/frame-2.fits.fz') + status.lowFluxAmplitudeWarning(18, 123.4, 234.5) + status.setCurrentFilename('/mnt/data/frame-3.fits.fz') + status.overexposedWarning(18, 124.4, 235.5, 58981.5) + status.setCurrentFilename('/mnt/data/frame-4.fits.fz') + status.skyBackgroundWarning(18, 125.4, 236.5) + + warning_messages = [message for message in messages if 'file frame-' in message] + assert len(warning_messages) == 4 + assert all('Tracked VSX variable DI Her' in message for message in warning_messages) + assert all('Comparison star' not in message for message in warning_messages) + assert all('/mnt/data/' not in message for message in warning_messages) + + +def test_repeated_frame_warnings_are_aggregated_with_progress_and_summary(): + messages = [] + status = PlateStatus(lambda message, **kwargs: messages.append(message)) + + for frame_index in range(205): + status.setCurrentFilename(f'/mnt/data/frame-{frame_index:04d}.fits') + status.overexposedWarning(1, 100.0, 200.0, 50000.0) + + status.logAggregatedWarningSummary() + + assert sum( + 'Comparison star #1 is overexposed in file' in message + for message in messages + ) == 1 + assert any( + 'Comparison star #1: overexposed in 100 frame(s) so far.' in message + for message in messages + ) + assert any( + 'Comparison star #1: overexposed in 200 frame(s) so far.' in message + for message in messages + ) + assert messages[-1] == '>-- Comparison star #1: overexposed in 205 frame(s).' + assert sum( + 'overexposed_comp1' in frame_status + for frame_status in status.statusByFilename.values() + ) == 205 + + message_count = len(messages) + status.logAggregatedWarningSummary() + assert len(messages) == message_count + + +def test_aggregated_warnings_preserve_exact_per_frame_csv_flags(tmp_path): + messages = [] + filenames = [f'/mnt/data/frame-{frame_index:04d}.fits' for frame_index in range(3)] + status = PlateStatus(lambda message, **kwargs: messages.append(message)) + status.initializeFilenames(filenames) + status.initializeComparisonStarCount(1) + + for filename in filenames: + status.setCurrentFilename(filename) + status.overexposedWarning(1, 100.0, 200.0, 50000.0) + + output_path = tmp_path / 'PlateStatus.csv' + status.writePlateStatus(output_path) + + with output_path.open(newline='') as handle: + rows = list(csv.reader(handle)) + header = rows[0] + overexposed_column = header.index('overexposed_comp1') + assert [row[overexposed_column] for row in rows[1:]] == ['True', 'True', 'True'] + assert sum( + 'Comparison star #1 is overexposed in file' in message + for message in messages + ) == 1 + assert messages[-1] == '>-- Comparison star #1: overexposed in 3 frame(s).' diff --git a/tests/test_plots.py b/tests/test_plots.py new file mode 100644 index 00000000..f6e537b7 --- /dev/null +++ b/tests/test_plots.py @@ -0,0 +1,905 @@ +import matplotlib +matplotlib.use("Agg") + +from types import SimpleNamespace + +import numpy as np +import matplotlib.pyplot as plt +import pytest +from matplotlib.axes import Axes + +from exotic.plots import ( + _format_parameter_value, + _short_ktmf_label, + plot_fov, + plot_adaptive_aperture_diagnostics, + plot_comp_star_candidate_lightcurve_fits, + plot_final_lightcurve, + plot_individual_comp_star_calibration_series, + plot_ktmf_qc_metrics, + plot_obs_stats, + plot_prior_posterior_comparison, + plot_differential_magnitude, + plot_stellar_variability, +) + + +class DummyFit: + def __init__(self): + self.time = np.array([1.0, 2.0, 3.0]) + self.airmass = np.array([1.1, 1.2, 1.3]) + + +def test_format_parameter_value_uses_uncertainty_precision_without_scientific_notation(): + assert ( + _format_parameter_value(2461197.8645824, 0.0005037355680314821, split_error=True) + == "2461197.86458\n+/- 0.00050" + ) + assert _format_parameter_value(89.3511, 2.16, unit="deg") == "89.4 +/- 2.2 deg" + + +def test_plot_obs_stats_applies_relative_flux_mask(tmp_path, monkeypatch): + fit = DummyFit() + psf_rows = np.arange(35, dtype=float).reshape(5, 7) + psf = {"target": psf_rows} + si = np.array([2, 0, 4, 1, 3]) + gi = np.array([True, False, True, True, True]) + relative_flux_mask = np.array([True, False, True, True]) + captured = [] + + original_plot = Axes.plot + + def spy_plot(self, x, y, *args, **kwargs): + captured.append((np.asarray(x), np.asarray(y))) + return original_plot(self, x, y, *args, **kwargs) + + monkeypatch.setattr(Axes, "plot", spy_plot) + + plot_obs_stats( + fit, + [], + psf, + si, + gi, + "Target", + str(tmp_path), + "2026-03-09", + relative_flux_mask=relative_flux_mask, + ) + + assert captured + np.testing.assert_array_equal(captured[0][0], fit.time) + np.testing.assert_array_equal(captured[0][1], np.array([14.0, 7.0, 21.0])) + assert (tmp_path / "working_artifacts" / "Observing_Statistics_target_2026-03-09.png").exists() + + +def test_stellar_variability_final_lightcurve_plots_calibrated_magnitude_by_time(tmp_path, monkeypatch): + fit = SimpleNamespace( + stellar_variability_only=True, + time=np.array([2461229.5, 2461229.6, 2461229.8]), + detrended=np.array([1.0, 1.01, 0.99]), + detrendederr=np.array([0.001, 0.001, 0.001]), + time_upsample=np.array([2461229.5, 2461229.8]), + transit_upsample=np.ones(2), + stellar_variability_params=[ + { + "time": 2461229.5, + "mag": 13.738, + "mag_err": 0.004, + "cmag": 13.739, + "cmag_err": 0.001, + "mag_band": "r", + "observed_filter": "SR", + "comp_ra": 295.3085, + "comp_dec": 56.1606, + "cname": "RA=295.3085000 Dec=56.1606000", + }, + { + "time": 2461229.6, + "mag": 13.740, + "mag_err": 0.004, + "cmag": 13.739, + "cmag_err": 0.001, + "mag_band": "r", + "observed_filter": "SR", + "comp_ra": 295.3085, + "comp_dec": 56.1606, + "cname": "RA=295.3085000 Dec=56.1606000", + }, + { + "time": 2461229.8, + "mag": 13.735, + "mag_err": 0.004, + "cmag": 13.739, + "cmag_err": 0.001, + "mag_band": "r", + "observed_filter": "SR", + "comp_ra": 295.3085, + "comp_dec": 56.1606, + "cname": "RA=295.3085000 Dec=56.1606000", + }, + ], + stellar_variability_target_name="WASP-194", + ) + captured_errorbar_x = [] + captured_errorbar_y = [] + captured_xlabels = [] + captured_ylabels = [] + inverted_axes = [] + + original_errorbar = Axes.errorbar + original_set_xlabel = Axes.set_xlabel + original_set_ylabel = Axes.set_ylabel + original_invert_yaxis = Axes.invert_yaxis + + def spy_errorbar(self, x, y, *args, **kwargs): + captured_errorbar_x.append(np.asarray(x, dtype=float)) + captured_errorbar_y.append(np.asarray(y, dtype=float)) + return original_errorbar(self, x, y, *args, **kwargs) + + def spy_set_xlabel(self, xlabel, *args, **kwargs): + captured_xlabels.append(xlabel) + return original_set_xlabel(self, xlabel, *args, **kwargs) + + def spy_set_ylabel(self, ylabel, *args, **kwargs): + captured_ylabels.append(ylabel) + return original_set_ylabel(self, ylabel, *args, **kwargs) + + def spy_invert_yaxis(self, *args, **kwargs): + inverted_axes.append(self) + return original_invert_yaxis(self, *args, **kwargs) + + monkeypatch.setattr(Axes, "errorbar", spy_errorbar) + monkeypatch.setattr(Axes, "set_xlabel", spy_set_xlabel) + monkeypatch.setattr(Axes, "set_ylabel", spy_set_ylabel) + monkeypatch.setattr(Axes, "invert_yaxis", spy_invert_yaxis) + + plot_final_lightcurve(fit, np.ones(2), "Target", str(tmp_path), "2026-07-08") + + np.testing.assert_allclose(captured_errorbar_x[-1], np.array([2461229.5, 2461229.6, 2461229.8])) + np.testing.assert_allclose(captured_errorbar_y[-1], np.array([13.738, 13.740, 13.735])) + assert captured_xlabels[-1] == "Time [BJD_TDB]" + assert captured_xlabels[-1] != "Orbital Phase" + assert captured_ylabels[-1] == "Magnitude (r)" + assert len(inverted_axes) == 2 + assert "O-C [%]" not in captured_ylabels + assert (tmp_path / "FinalLightCurve_Target_2026-07-08.png").exists() + + +def test_stellar_variability_differential_plot_survives_without_apparent_magnitudes( + tmp_path, monkeypatch): + from matplotlib.axes import Axes + + captured_titles = [] + original_set_title = Axes.set_title + + def spy_set_title(self, title, *args, **kwargs): + captured_titles.append(title) + return original_set_title(self, title, *args, **kwargs) + + monkeypatch.setattr(Axes, "set_title", spy_set_title) + fit = SimpleNamespace( + stellar_variability_only=True, + time=np.array([2461229.5, 2461229.6, 2461229.8]), + data=np.ones(3), + dataerr=np.full(3, 0.001), + airmass=np.array([1.1, 1.2, 1.3]), + airmass_model=np.array([0.8, 1.0, 1.2]), + transit=np.ones(3), + stellar_variability_target_flux=np.array([900.0, 1000.0, 1100.0]), + stellar_variability_comp_flux=np.full(3, 1000.0), + stellar_variability_target_flux_error=np.ones(3), + stellar_variability_comp_flux_error=np.ones(3), + stellar_variability_params=[], + ) + + output_path = plot_differential_magnitude( + fit, + 'Variable Star', + tmp_path, + '2026-08-02', + observed_filter='V', + ) + + assert output_path.exists() + assert (tmp_path / 'Stellar_Variability_DifferentialMagnitude.png').exists() + assert ( + tmp_path / 'working_artifacts' / 'Stellar_Variability_DifferentialMagnitude.png' + ).exists() + assert captured_titles[-1] == 'Variable Star' + + +def test_plot_obs_stats_uses_supplied_background_series(tmp_path, monkeypatch): + fit = DummyFit() + psf_rows = np.arange(35, dtype=float).reshape(5, 7) + psf = {"target": psf_rows} + si = np.array([2, 0, 4, 1, 3]) + gi = np.array([True, False, True, True, True]) + relative_flux_mask = np.array([True, False, True, True]) + background_series = {"target": np.array([100.0, 200.0, 300.0, 400.0, 500.0])} + captured = [] + + original_plot = Axes.plot + + def spy_plot(self, x, y, *args, **kwargs): + captured.append((np.asarray(x), np.asarray(y))) + return original_plot(self, x, y, *args, **kwargs) + + monkeypatch.setattr(Axes, "plot", spy_plot) + + plot_obs_stats( + fit, + [], + psf, + si, + gi, + "Target", + str(tmp_path), + "2026-03-09", + relative_flux_mask=relative_flux_mask, + background_series=background_series, + ) + + assert len(captured) >= 6 + np.testing.assert_array_equal(captured[5][0], fit.time) + np.testing.assert_array_equal(captured[5][1], np.array([300.0, 200.0, 400.0])) + + +def test_plot_adaptive_aperture_diagnostics_writes_outputs(tmp_path): + plot_adaptive_aperture_diagnostics( + times=np.array([1.0, 2.0, 3.0]), + aperture_series=np.array([7.5, 8.0, 8.5]), + annulus_series=np.array([25.0, 26.0, 27.0]), + fwhm_series=np.array([3.0, 3.2, 3.4]), + airmass=np.array([1.1, 1.2, 1.3]), + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + aperture_sigma=2.5, + annulus_sigma=9.0, + ) + + assert (tmp_path / "working_artifacts" / "AdaptiveApertureDiagnostics_Target_2026-03-09.png").exists() + assert (tmp_path / "working_artifacts" / "AdaptiveApertureDiagnostics_Target_2026-03-09.pdf").exists() + + +def test_plot_fov_psf_legend_omits_aperture_annulus_text(tmp_path, monkeypatch): + labels = [] + + original_legend = plt.legend + + def spy_legend(*args, **kwargs): + legend = original_legend(*args, **kwargs) + labels.extend(text.get_text() for text in legend.get_texts()) + return legend + + monkeypatch.setattr(plt, "legend", spy_legend) + + plot_fov( + aper=20.0, + annulus=60.0, + sigma=4.0, + x_targ=50.0, + y_targ=60.0, + x_ref=90.0, + y_ref=100.0, + image=np.ones((200, 200)), + image_scale="Image scale in arcsec/pixel: 0.53", + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + opt_method="PSF", + min_aper_fov=20.44, + min_annulus_fov=61.31, + ) + + assert labels + assert set(labels) == {"PSF Photometry"} + + +def test_plot_fov_marks_every_ensemble_comparison(tmp_path, monkeypatch): + plotted_labels = [] + original_text = Axes.text + + def spy_text(self, x, y, text, *args, **kwargs): + plotted_labels.append(text) + return original_text(self, x, y, text, *args, **kwargs) + + monkeypatch.setattr(Axes, "text", spy_text) + + plot_fov( + aper=8.0, + annulus=20.0, + sigma=2.0, + x_targ=50.0, + y_targ=60.0, + x_ref=90.0, + y_ref=100.0, + image=np.ones((220, 220)), + image_scale="Image scale in arcsec/pixel: 0.53", + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + opt_method="Aperture", + min_aper_fov=8.0, + min_annulus_fov=20.0, + comparison_positions=[[90.0, 100.0], [130.0, 140.0], [170.0, 180.0]], + comparison_labels=["Comp 1", "Comp 3", "Comp 4"], + ) + + assert {"Target", "Comp 1", "Comp 3", "Comp 4"}.issubset(plotted_labels) + assert ( + tmp_path / "working_artifacts" / "FOV_Target_LinearStretch_2026-03-09.png" + ).is_file() + + +def test_plot_individual_comp_star_calibration_series_writes_outputs(tmp_path): + plot_individual_comp_star_calibration_series( + times=np.array([1.0, 2.0, 3.0]), + comp_summaries=[ + { + "label": "Comp 1", + "selected": True, + "aggregate_score": 0.0012, + "pairwise_ratio_series": {"vs 2": np.array([1.0, 1.01, 0.99])}, + "ensemble_ratio_series": np.array([1.0, 1.005, 0.995]), + }, + { + "label": "Comp 2", + "selected": False, + "aggregate_score": 0.0025, + "pairwise_ratio_series": {"vs 1": np.array([0.99, 1.0, 1.01])}, + "ensemble_ratio_series": np.array([0.995, 1.0, 1.005]), + }, + ], + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + method_label="PSF photometry", + ) + + assert (tmp_path / "working_artifacts" / "CompStarCalibrationCurve_Comp1_Target_2026-03-09.png").exists() + assert (tmp_path / "working_artifacts" / "CompStarCalibrationCurve_Comp1_Target_2026-03-09.pdf").exists() + assert (tmp_path / "working_artifacts" / "CompStarCalibrationCurve_Comp2_Target_2026-03-09.png").exists() + assert (tmp_path / "working_artifacts" / "CompStarCalibrationCurve_Comp2_Target_2026-03-09.pdf").exists() + + +def test_plot_individual_comp_star_calibration_series_masks_rejected_frame_lines(tmp_path, monkeypatch): + captured_lines = {} + original_plot = Axes.plot + + def spy_plot(self, x, y, *args, **kwargs): + label = kwargs.get("label") + if label in {"vs 2", "Intercomparison"}: + captured_lines[label] = (np.asarray(x), np.asarray(y)) + return original_plot(self, x, y, *args, **kwargs) + + monkeypatch.setattr(Axes, "plot", spy_plot) + + plot_individual_comp_star_calibration_series( + times=np.array([1.0, 2.0, 3.0, 4.0]), + comp_summaries=[ + { + "label": "Comp 1", + "selected": False, + "aggregate_score": 0.01, + "pairwise_ratio_series": {"vs 2": np.array([1.0, 0.05, 1.01, 0.99])}, + "ensemble_ratio_series": np.array([1.0, 0.02, 1.005, 0.995]), + "ensemble_frame_keep_mask": np.array([True, False, True, True]), + }, + ], + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + method_label="Aperture photometry", + ) + + assert np.isnan(captured_lines["vs 2"][1][1]) + assert np.isnan(captured_lines["Intercomparison"][1][1]) + + +def test_plot_stellar_variability_labels_reference_coordinates(tmp_path, monkeypatch): + titles = [] + ylabels = [] + inverted_axes = [] + original_set_title = Axes.set_title + original_set_ylabel = Axes.set_ylabel + original_invert_yaxis = Axes.invert_yaxis + + def spy_set_title(self, label, *args, **kwargs): + titles.append(label) + return original_set_title(self, label, *args, **kwargs) + + def spy_set_ylabel(self, label, *args, **kwargs): + ylabels.append(label) + return original_set_ylabel(self, label, *args, **kwargs) + + def spy_invert_yaxis(self, *args, **kwargs): + inverted_axes.append(self) + return original_invert_yaxis(self, *args, **kwargs) + + monkeypatch.setattr(Axes, "set_title", spy_set_title) + monkeypatch.setattr(Axes, "set_ylabel", spy_set_ylabel) + monkeypatch.setattr(Axes, "invert_yaxis", spy_invert_yaxis) + + plot_stellar_variability( + [ + { + "time": 2450000.1, + "mag": 12.34, + "mag_err": 0.05, + "cmag": 12.345, + "cmag_err": 0.067, + "comp_ra": 10.1, + "comp_dec": -20.2, + "mag_band": "r", + "observed_filter": "CV", + "is_aavso_vsp": False, + } + ], + str(tmp_path), + "Host Star", + "NextAstro-123", + ) + + assert titles[-1] == ( + "Host Star\n" + "Label: NextAstro-123\n" + "Comparison RA=10.100000\n" + "Dec=-20.200000\n" + "Original filter: CV | Comparison mag: r=12.3450 +/- 0.0670" + ) + assert ylabels[-1] == "Magnitude (r)" + assert len(inverted_axes) == 1 + assert (tmp_path / "working_artifacts" / "Stellar_Variability.png").exists() + assert (tmp_path / "Stellar_Variability.png").exists() + + +def test_plot_stellar_variability_labels_aavso_filter_and_assumed_comparison(tmp_path, monkeypatch): + titles = [] + ylabels = [] + original_set_title = Axes.set_title + original_set_ylabel = Axes.set_ylabel + + def spy_set_title(self, label, *args, **kwargs): + titles.append(label) + return original_set_title(self, label, *args, **kwargs) + + def spy_set_ylabel(self, label, *args, **kwargs): + ylabels.append(label) + return original_set_ylabel(self, label, *args, **kwargs) + + monkeypatch.setattr(Axes, "set_title", spy_set_title) + monkeypatch.setattr(Axes, "set_ylabel", spy_set_ylabel) + + plot_stellar_variability( + [ + { + "time": 2450000.1, + "mag": 12.34, + "mag_err": 0.05, + "cmag": 12.345, + "cmag_err": 0.067, + "comp_ra": 10.1, + "comp_dec": -20.2, + "mag_band": "ClearV", + "catalog_mag_band": "V", + "observed_filter": "CV", + "is_aavso_vsp": True, + } + ], + str(tmp_path), + "Host Star", + "000-BJX-718", + ) + + assert ( + "Label: 000-BJX-718\n" + "Comparison RA=10.100000\n" + "Dec=-20.200000" + ) in titles[-1] + assert "Original filter: CV" in titles[-1] + assert "Comparison mag: V=12.3450 +/- 0.0670" in titles[-1] + assert ylabels[-1] == "Magnitude (ClearV)" + + +def test_plot_stellar_variability_omits_invalid_reference_magnitudes(tmp_path, monkeypatch): + titles = [] + original_set_title = Axes.set_title + + def spy_set_title(self, label, *args, **kwargs): + titles.append(label) + return original_set_title(self, label, *args, **kwargs) + + monkeypatch.setattr(Axes, "set_title", spy_set_title) + + plot_stellar_variability( + [ + { + "time": 2450000.1, + "mag": 12.34, + "mag_err": 0.05, + "cmag": 99.99, + "cmag_err": 99.99, + "comp_ra": 10.1, + "comp_dec": -20.2, + "mag_band": "V", + "observed_filter": "MObs CV", + "is_aavso_vsp": False, + } + ], + str(tmp_path), + "Host Star", + "NextAstro-123", + ) + + assert titles[-1] == ( + "Host Star\n" + "Label: NextAstro-123\nComparison RA=10.100000\nDec=-20.200000\n" + "Original filter: MObs CV" + ) + assert "99.99" not in titles[-1] + assert "V=" not in titles[-1] + + +def test_plot_stellar_variability_skips_over_30_measurements(tmp_path): + plot_stellar_variability( + [ + { + "time": 2450000.1, + "mag": 99.99, + "mag_err": 0.05, + "cmag": 12.0, + "cmag_err": 0.05, + "mag_band": "V", + } + ], + str(tmp_path), + "Host Star", + "Comp", + ) + + assert not (tmp_path / "working_artifacts" / "Stellar_Variability.png").exists() + assert not (tmp_path / "Stellar_Variability.png").exists() + + +def test_plot_comp_star_candidate_lightcurve_fits_writes_outputs(tmp_path): + class DummyCandidateFit: + def __init__(self): + self.kwargs = None + + def plot_bestfit(self, phase=False, show_flux_baseline_label=True): + self.kwargs = { + "phase": phase, + "show_flux_baseline_label": show_flux_baseline_label, + } + fig, axes = plt.subplots(2, 1) + return fig, axes + selected_fit = DummyCandidateFit() + other_fit = DummyCandidateFit() + + plot_comp_star_candidate_lightcurve_fits( + candidate_fit_summaries=[ + {"label": "Comp 1", "selected": True, "fit": selected_fit, "res_std": 0.0012}, + {"label": "Comp 2", "selected": False, "fit": other_fit, "res_std": 0.0025}, + {"label": "Comp 3", "selected": False, "fit": None, "res_std": np.inf}, + ], + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + method_label="Aperture photometry (aper=5.00px, annulus=12.00px)", + ) + + assert selected_fit.kwargs == {"phase": False, "show_flux_baseline_label": False} + assert other_fit.kwargs == {"phase": False, "show_flux_baseline_label": False} + assert (tmp_path / "working_artifacts" / "CompStarLightCurveFit_Comp1_Target_2026-03-09.png").exists() + assert (tmp_path / "working_artifacts" / "CompStarLightCurveFit_Comp1_Target_2026-03-09.pdf").exists() + assert (tmp_path / "working_artifacts" / "CompStarLightCurveFit_Comp2_Target_2026-03-09.png").exists() + assert (tmp_path / "working_artifacts" / "CompStarLightCurveFit_Comp2_Target_2026-03-09.pdf").exists() + assert not (tmp_path / "working_artifacts" / "CompStarLightCurveFit_Comp3_Target_2026-03-09.png").exists() + + +def test_plot_final_lightcurve_requests_uncertainty_bands_without_baseline_label(tmp_path): + class DummyFinalFit: + def __init__(self): + self.kwargs = None + self.phase_upsample = np.linspace(-0.05, 0.05, 5) + self.transit_upsample = np.ones(5) + + def plot_bestfit(self, show_flux_baseline_label=True, show_model_uncertainty=False, + show_baseline_uncertainty=False): + self.kwargs = { + "show_flux_baseline_label": show_flux_baseline_label, + "show_model_uncertainty": show_model_uncertainty, + "show_baseline_uncertainty": show_baseline_uncertainty, + } + fig, axes = plt.subplots(2, 1) + return fig, axes + + fit = DummyFinalFit() + + plot_final_lightcurve( + fit, + high_res=np.ones(5), + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + ) + + assert fit.kwargs == { + "show_flux_baseline_label": False, + "show_model_uncertainty": True, + "show_baseline_uncertainty": True, + } + assert (tmp_path / "FinalLightCurve_Target_2026-03-09.png").exists() + assert (tmp_path / "FinalLightCurve_Target_2026-03-09.pdf").exists() + + +def test_plot_final_lightcurve_adds_apparent_magnitude_axis_when_calibrated(tmp_path, monkeypatch): + class DummyFinalFit: + def __init__(self): + self.kwargs = None + self.phase_upsample = np.linspace(-0.05, 0.05, 5) + self.transit_upsample = np.ones(5) + self.stellar_variability_params = [ + {"time": 1.0, "mag": 13.739, "mag_err": 0.001, "mag_band": "r"}, + {"time": 2.0, "mag": 13.741, "mag_err": 0.002, "mag_band": "r"}, + ] + + def plot_bestfit(self, show_flux_baseline_label=True, show_model_uncertainty=False, + show_baseline_uncertainty=False): + self.kwargs = { + "show_flux_baseline_label": show_flux_baseline_label, + "show_model_uncertainty": show_model_uncertainty, + "show_baseline_uncertainty": show_baseline_uncertainty, + } + fig, axes = plt.subplots(2, 1) + return fig, axes + + secondary_calls = [] + secondary_labels = [] + + class FakeSecondaryAxis: + def set_ylabel(self, label): + secondary_labels.append(label) + + def spy_secondary_yaxis(self, location, functions=None, *args, **kwargs): + secondary_calls.append((location, functions)) + return FakeSecondaryAxis() + + monkeypatch.setattr(Axes, "secondary_yaxis", spy_secondary_yaxis) + + plot_final_lightcurve( + DummyFinalFit(), + high_res=np.ones(5), + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + ) + + assert secondary_labels == ["Apparent Magnitude (r)"] + assert secondary_calls[0][0] == "right" + flux_to_mag, mag_to_flux = secondary_calls[0][1] + assert flux_to_mag(np.array([1.0])) == pytest.approx(np.array([13.740])) + assert mag_to_flux(np.array([13.740])) == pytest.approx(np.array([1.0])) + + +def test_plot_final_lightcurve_draws_data_scatter_uncertainty_band(tmp_path, monkeypatch): + captured = [] + original_fill_between = Axes.fill_between + + def spy_fill_between(self, x, y1, y2=0, *args, **kwargs): + captured.append({ + "x": np.asarray(x, dtype=float), + "y1": np.asarray(y1, dtype=float), + "y2": np.asarray(y2, dtype=float), + "color": kwargs.get("color"), + "alpha": kwargs.get("alpha"), + "label": kwargs.get("label"), + }) + return original_fill_between(self, x, y1, y2, *args, **kwargs) + + monkeypatch.setattr(Axes, "fill_between", spy_fill_between) + + class DummyFinalFit: + def __init__(self): + self.phase_upsample = np.linspace(-0.05, 0.05, 41) + depth_shape = np.exp(-0.5 * (self.phase_upsample / 0.015) ** 2) + self.transit_upsample = 1.0 - 0.01 * depth_shape + self.time_upsample = self.phase_upsample.copy() + self.phase = self.phase_upsample.copy() + self.transit = self.transit_upsample.copy() + self.model = self.transit.copy() + residual_pattern = 0.02 * np.sin(np.linspace(0, 6 * np.pi, self.model.size)) + self.data = self.model + residual_pattern + self.residuals = self.data - self.model + self.dataerr = np.full_like(self.model, 0.02) + self.parameters = {"rprs": 0.1} + self.errors = {"rprs": 0.001} + + def transit_model_uncertainty(self, times): + return self.transit_upsample - 0.001, self.transit_upsample + 0.001 + + def plot_bestfit(self, show_flux_baseline_label=True, show_model_uncertainty=False, + show_baseline_uncertainty=False): + fig, axes = plt.subplots(2, 1) + axes[0].plot(self.phase_upsample, self.transit_upsample, 'r-', label='model') + axes[0].legend(loc='best') + return fig, axes + + plot_final_lightcurve( + DummyFinalFit(), + high_res=np.ones(41), + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + ) + + purple_bands = [item for item in captured if item["color"] == "#6a1b9a"] + assert purple_bands + assert all(item["label"] == "_nolegend_" for item in purple_bands) + assert all(item["alpha"] <= 0.16 for item in purple_bands) + assert any(np.nanmax(np.abs(item["y2"] - item["y1"])) > 0.001 for item in purple_bands) + + +def test_plot_final_lightcurve_marks_final_residual_rejections(tmp_path, monkeypatch): + captured = [] + original_scatter = Axes.scatter + + def spy_scatter(self, x, y, *args, **kwargs): + captured.append({ + "x": np.asarray(x, dtype=float), + "y": np.asarray(y, dtype=float), + "label": kwargs.get("label"), + "color": kwargs.get("color"), + }) + return original_scatter(self, x, y, *args, **kwargs) + + monkeypatch.setattr(Axes, "scatter", spy_scatter) + + class DummyFinalFit: + def __init__(self): + self.phase_upsample = np.linspace(-0.05, 0.05, 5) + self.transit_upsample = np.ones(5) + self.final_residual_rejection = { + "applied": True, + "rejected_phase": [0.01], + "rejected_flux": [0.92], + "rejected_residual_percent": [-8.0], + } + + def plot_bestfit(self, show_flux_baseline_label=True, show_model_uncertainty=False, + show_baseline_uncertainty=False): + fig, axes = plt.subplots(2, 1) + return fig, axes + + plot_final_lightcurve( + DummyFinalFit(), + high_res=np.ones(5), + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + ) + + residual_rejection_points = [ + item for item in captured + if item["label"] == "_nolegend_" and item["color"] == "red" + ] + assert len(residual_rejection_points) == 2 + np.testing.assert_allclose(residual_rejection_points[0]["x"], [0.01]) + np.testing.assert_allclose(residual_rejection_points[1]["y"], [-8.0]) + + +def test_plot_prior_posterior_comparison_omits_prior_fallback_rprs(tmp_path, monkeypatch): + captured_text = [] + original_text = Axes.text + + def spy_text(self, x, y, s, *args, **kwargs): + captured_text.append(str(s)) + return original_text(self, x, y, s, *args, **kwargs) + + monkeypatch.setattr(Axes, "text", spy_text) + + class DummyFit: + def __init__(self): + self.parameters = { + "tmid": 1.012, + "rprs": 0.1, + "ars": 10.6, + "inc": 88.2, + } + self.errors = { + "tmid": 0.002, + "rprs": 0.005, + "ars": 0.4, + "inc": 0.3, + } + self.rprs_prior_fallback_applied = True + self.empirical_transit_uncertainty = { + "available": True, + "combined_rprs_uncertainty": 0.02, + "rprs_uncertainty_basis": "prior_assumed_data_only", + } + + planet_dict = { + "midT": 1.0, + "midTUnc": 0.001, + "rprs": 0.1, + "rprsUnc": 0.003, + "aRs": 10.0, + "aRsUnc": 0.2, + "inc": 89.0, + "incUnc": 0.4, + } + + output = plot_prior_posterior_comparison( + DummyFit(), + planet_dict, + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + ) + + assert output == tmp_path / "PriorPosteriorComparison_Target_2026-03-09.png" + assert output.exists() + assert (tmp_path / "PriorPosteriorComparison_Target_2026-03-09.pdf").exists() + assert any("Rp/R* omitted: prior value assumed, not measured" in text for text in captured_text) + assert any("Prior\n" in text and "Posterior\n" in text for text in captured_text) + assert not any("BJD_TDB" in text for text in captured_text) + assert not any("Prior epoch" in text for text in captured_text) + assert not any("Posterior (Prior)" in text for text in captured_text) + + +def test_plot_ktmf_qc_metrics_writes_outputs_and_annotations(tmp_path, monkeypatch): + captured_text = [] + original_text = Axes.text + + def spy_text(self, x, y, s, *args, **kwargs): + captured_text.append(str(s)) + return original_text(self, x, y, s, *args, **kwargs) + + monkeypatch.setattr(Axes, "text", spy_text) + + class DummyKTMFFit: + def __init__(self): + self.transit_qc = { + "status": "fail", + "ktmf_metric": 3.07, + "ktmf_contributions": [ + { + "label": "EEBLS Depth SNR", + "available": True, + "points": 0.11, + "max_points": 0.89, + "score": 0.12, + "detail": "2.00", + }, + { + "label": "Residual Scatter Around Full Model Fit", + "available": True, + "points": 0.14, + "max_points": 0.78, + "score": 0.18, + "detail": "2.3437%", + }, + ], + } + + output = plot_ktmf_qc_metrics( + DummyKTMFFit(), + targ_name="Target", + save=str(tmp_path), + date="2026-03-09", + ) + + assert output == tmp_path / "KTMF_QC_Target_2026-03-09.png" + assert output.exists() + assert (tmp_path / "KTMF_QC_Target_2026-03-09.pdf").exists() + assert any("KTMF\n3.07 / 5.00\nMARGINAL" in text for text in captured_text) + assert any("0.11 / 0.89" in text for text in captured_text) + assert not any("2.00" in text for text in captured_text) + assert not any("2.3437" in text for text in captured_text) + + +def test_ktmf_plot_shortens_tmid_posterior_gaussianity_label(): + assert _short_ktmf_label("Tmid Posterior Gaussianity") == "Tmid Gaussianity" diff --git a/tests/test_plotting_contours.py b/tests/test_plotting_contours.py new file mode 100644 index 00000000..537e630d --- /dev/null +++ b/tests/test_plotting_contours.py @@ -0,0 +1,17 @@ +import numpy as np + +from exotic.api import plotting + + +def test_contour_levels_inside_surface_are_preserved(): + levels = plotting._contour_levels_within_surface([0.2, 0.5, 0.8], 0.0, 1.0) + + np.testing.assert_allclose(levels, np.array([0.2, 0.5, 0.8])) + + +def test_contour_levels_outside_surface_are_clipped_into_drawable_range(): + levels = plotting._contour_levels_within_surface([10.0, 20.0, 30.0], 0.0, 1.0) + + assert levels.size == 1 + assert 0.0 < levels[0] < 1.0 + diff --git a/tests/test_radec_non_interactive.py b/tests/test_radec_non_interactive.py new file mode 100644 index 00000000..51f71ef9 --- /dev/null +++ b/tests/test_radec_non_interactive.py @@ -0,0 +1,75 @@ +import pytest + +from exotic import exotic as exotic_module + + +def test_invalid_target_coordinates_use_nasa_archive_fallback_without_prompt(monkeypatch): + messages = [] + monkeypatch.setattr( + 'builtins.input', + lambda prompt: pytest.fail("non-interactive coordinate resolution must not prompt"), + ) + monkeypatch.setattr( + exotic_module, + 'log_info', + lambda message, **kwargs: messages.append((message, kwargs)), + ) + + ra, dec = exotic_module.radec_hours_to_degree( + 'not-an-ra', + '+20:00:00', + non_interactive_run=True, + archive_ra=123.456, + archive_dec=-45.678, + target_name='Example b', + ) + + assert ra == pytest.approx(123.456) + assert dec == pytest.approx(-45.678) + assert len(messages) == 1 + assert "Using NASA Exoplanet Archive coordinates" in messages[0][0] + assert messages[0][1] == {'warn': True} + + +def test_invalid_target_and_archive_coordinates_abort_without_prompt(monkeypatch): + monkeypatch.setattr( + 'builtins.input', + lambda prompt: pytest.fail("non-interactive coordinate resolution must not prompt"), + ) + + with pytest.raises( + ValueError, + match=( + r"Non-interactive run cancelled for target Example b: .*" + r"NASA Exoplanet Archive coordinates .* are also unusable" + ), + ): + exotic_module.radec_hours_to_degree( + 'not-an-ra', + '+20:00:00', + non_interactive_run=True, + archive_ra='also-not-an-ra', + archive_dec='also-not-a-dec', + target_name='Example b', + ) + + +def test_invalid_target_coordinates_abort_when_archive_coordinates_unavailable(monkeypatch): + monkeypatch.setattr( + 'builtins.input', + lambda prompt: pytest.fail("non-interactive coordinate resolution must not prompt"), + ) + + with pytest.raises( + ValueError, + match=( + r"Non-interactive run cancelled for target Example b: .*" + r"NASA Exoplanet Archive coordinates are unavailable" + ), + ): + exotic_module.radec_hours_to_degree( + 'not-an-ra', + '+20:00:00', + non_interactive_run=True, + target_name='Example b', + ) diff --git a/tests/test_runtime_timing.py b/tests/test_runtime_timing.py new file mode 100644 index 00000000..a5fc70a3 --- /dev/null +++ b/tests/test_runtime_timing.py @@ -0,0 +1,29 @@ +from datetime import datetime + +import pytest + +from exotic import exotic as exotic_module + + +def test_format_clock_log_message_uses_hour_and_minute_and_preserves_leading_newlines(): + message = exotic_module.format_clock_log_message( + "\n\nStarting reduction", + clock_time=datetime(2026, 7, 16, 7, 5, 42), + ) + + assert message == "\n\n[07:05] Starting reduction" + + +def test_reduction_stage_timer_logs_step_and_total_elapsed_seconds(monkeypatch): + clock_values = iter([100.0, 102.5, 109.0]) + logged = [] + monkeypatch.setattr(exotic_module, "log_info", logged.append) + + timer = exotic_module.ReductionStageTimer(time_source=lambda: next(clock_values)) + + assert timer.checkpoint("first stage") == pytest.approx(2.5) + assert timer.checkpoint("second stage") == pytest.approx(6.5) + assert logged == [ + "STEP TIMING | first stage | elapsed_s=2.50 | total_s=2.50", + "STEP TIMING | second stage | elapsed_s=6.50 | total_s=9.00", + ] diff --git a/tests/test_transit_duration.py b/tests/test_transit_duration.py new file mode 100644 index 00000000..9ce1dc23 --- /dev/null +++ b/tests/test_transit_duration.py @@ -0,0 +1,111 @@ +"""Tests for the analytic transit-duration formula (issue #1383). + +The eccentric duration must follow Winn (2010), "Transits and Occultations", +arXiv:1001.2010: the impact parameter uses the eccentricity factor +(1 - e^2)/(1 + e sin w) (eq. 7), the arcsin argument is normalized by the +plain a/R* (eq. 14), and the whole expression is multiplied by the velocity +factor sqrt(1 - e^2)/(1 + e sin w) (eq. 16). Putting the eq.-7 factor inside +the arcsin instead inverts the eccentricity dependence: periastron-at-transit +(fastest planet, shortest transit) comes out longest, and the error grows as +(1 + e sin w)^2 / (1 - e^2)^{3/2}, reaching ~60x for Kepler-1704 b. +""" + +import math + +import numpy as np +import pytest + +from exotic.transit_depth import transit_duration_days + + +def winn_2010_duration_days(period, ars, inc_deg, rprs, ecc, omega_deg): + """Reference implementation: Winn (2010) eqs. 7, 14, 16.""" + inc = math.radians(inc_deg) + omega = math.radians(omega_deg) + denom = 1.0 + ecc * math.sin(omega) + b = ars * math.cos(inc) * (1.0 - ecc ** 2) / denom + chord_sq = (1.0 + rprs) ** 2 - b ** 2 + if chord_sq <= 0: + return float("nan") + argument = min(math.sqrt(chord_sq) / (ars * math.sin(inc)), 1.0) + velocity_factor = math.sqrt(1.0 - ecc ** 2) / denom + return (period / math.pi) * math.asin(argument) * velocity_factor + + +def duration_params(period, ars, inc, rprs, ecc, omega): + return { + "per": period, + "ars": ars, + "inc": inc, + "rprs": rprs, + "ecc": ecc, + "omega": omega, + } + + +def test_circular_matches_winn_exactly(): + params = duration_params(3.0, 10.0, 90.0, 0.1, 0.0, 90.0) + expected = winn_2010_duration_days(3.0, 10.0, 90.0, 0.1, 0.0, 90.0) + assert transit_duration_days(params) == pytest.approx(expected, rel=1e-12) + + +@pytest.mark.parametrize("ecc", [0.1, 0.3, 0.5, 0.7, 0.9]) +@pytest.mark.parametrize("omega", [0.0, 45.0, 90.0, 135.0, 180.0, 270.0]) +@pytest.mark.parametrize("inc", [90.0, 88.0, 85.0]) +def test_eccentric_matches_winn(ecc, omega, inc): + params = duration_params(3.0, 10.0, inc, 0.1, ecc, omega) + expected = winn_2010_duration_days(3.0, 10.0, inc, 0.1, ecc, omega) + result = transit_duration_days(params) + if math.isnan(expected): + assert math.isnan(result) + else: + assert result == pytest.approx(expected, rel=1e-9) + + +def test_periastron_transit_is_shorter_and_apastron_longer(): + circular = transit_duration_days(duration_params(3.0, 10.0, 90.0, 0.1, 0.0, 90.0)) + periastron = transit_duration_days(duration_params(3.0, 10.0, 90.0, 0.1, 0.5, 90.0)) + apastron = transit_duration_days(duration_params(3.0, 10.0, 90.0, 0.1, 0.5, 270.0)) + assert periastron < circular < apastron + + +@pytest.mark.parametrize( + "name, period, ars, inc, rprs, ecc, omega, published_hours", + [ + # NASA Exoplanet Archive `ps` default rows, pl_trandur in hours. + ("Kepler-1704 b", 988.88112, 256.4, 89.00, 0.0644, 0.920, 82.40, 6.007), + ("HD 17156 b", 21.2164294, 23.11, 86.51, 0.07412, 0.6772, 122.06, 3.1505), + ("HD 80606 b", 111.436765, 94.452, 89.24, 0.1009, 0.93183, -58.887, 11.98), + ], +) +def test_reproduces_published_durations(name, period, ars, inc, rprs, ecc, omega, published_hours): + params = duration_params(period, ars, inc, rprs, ecc, omega) + hours = transit_duration_days(params) * 24.0 + assert hours == pytest.approx(published_hours, rel=0.05), name + + +def _exotic_main_module(): + return pytest.importorskip( + "exotic.exotic", reason="exotic.exotic imports the full pipeline dependency stack" + ) + + +def test_qc_contact_duration_matches_winn(): + exotic_main = _exotic_main_module() + params = {"per": 3.0, "ars": 10.0, "inc": 89.0, "ecc": 0.5, "omega": 90.0} + result = exotic_main.transit_qc_geometry_contact_duration(params, 1.1) + inc = math.radians(89.0) + omega = math.radians(90.0) + denom = 1.0 + 0.5 * math.sin(omega) + b = 10.0 * math.cos(inc) * (1.0 - 0.25) / denom + argument = min(math.sqrt(1.1 ** 2 - b ** 2) / (10.0 * math.sin(inc)), 1.0) + expected = (3.0 / math.pi) * math.asin(argument) * (math.sqrt(0.75) / denom) + assert result == pytest.approx(expected, rel=1e-9) + + +def test_prior_geometry_duration_matches_winn(): + exotic_main = _exotic_main_module() + prior = {"per": 3.0, "ars": 10.0, "inc": 88.0, "rprs": 0.1, "ecc": 0.4, "omega": 120.0} + result = exotic_main.estimate_transit_duration_from_prior_geometry(prior) + expected = winn_2010_duration_days(3.0, 10.0, 88.0, 0.1, 0.4, 120.0) + assert result == pytest.approx(expected, rel=1e-9) diff --git a/tests/test_ultranest_utils.py b/tests/test_ultranest_utils.py new file mode 100644 index 00000000..69d4a07d --- /dev/null +++ b/tests/test_ultranest_utils.py @@ -0,0 +1,690 @@ +import io +import logging +import sys +import types + +import numpy as np +import pytest + +import exotic.api.ultranest_utils as ultranest_utils +from exotic.api.ultranest_utils import run_reactive_sampler +from exotic.api.ultranest_utils import supports_ultranest_live_status + + +_MPI_ENV_KEYS = ( + "OMPI_COMM_WORLD_SIZE", + "PMI_SIZE", + "PMIX_SIZE", + "MV2_COMM_WORLD_SIZE", + "OMPI_COMM_WORLD_RANK", + "PMI_RANK", + "PMIX_RANK", + "MV2_COMM_WORLD_RANK", +) + + +class _FakeStream(io.StringIO): + def __init__(self, tty): + super().__init__() + self._tty = tty + + def isatty(self): + return self._tty + + +def _reset_ultranest_env(monkeypatch): + monkeypatch.delenv("EXOTIC_ULTRANEST_PLAIN_PROGRESS", raising=False) + monkeypatch.delenv("EXOTIC_ULTRANEST_RICH_PROGRESS", raising=False) + monkeypatch.delenv("EXOTIC_ULTRANEST_MIN_NUM_LIVE_POINTS", raising=False) + monkeypatch.delenv("EXOTIC_ULTRANEST_MIN_LIVE_POINTS", raising=False) + monkeypatch.delenv("EXOTIC_ULTRANEST_WORKERS", raising=False) + monkeypatch.delenv("EXOTIC_ULTRANEST_WORKER_BACKEND", raising=False) + monkeypatch.delenv("NEXTASTRO_EXOTIC_ULTRANEST_WORKERS", raising=False) + for env_key in _MPI_ENV_KEYS: + monkeypatch.delenv(env_key, raising=False) + monkeypatch.delenv("CI", raising=False) + + +def test_supports_ultranest_live_status_overrides(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("TERM", "xterm-256color") + stream = _FakeStream(tty=True) + + monkeypatch.setenv("EXOTIC_ULTRANEST_RICH_PROGRESS", "1") + assert supports_ultranest_live_status(stream=stream) is True + + monkeypatch.setenv("EXOTIC_ULTRANEST_PLAIN_PROGRESS", "1") + assert supports_ultranest_live_status(stream=stream) is False + + +def test_run_reactive_sampler_compat_mode_emits_progress(monkeypatch): + _reset_ultranest_env(monkeypatch) + stream = _FakeStream(tty=False) + + class FakeSampler: + def __init__(self): + self.kwargs = None + + def run(self, **kwargs): + self.kwargs = kwargs + callback = kwargs["viz_callback"] + callback(None, {"it": 120, "ncall": 540, "logz": -151.9, "logz_remain": -150.0}) + callback(None, {"it": 360, "ncall": 907, "logz": -139.2, "logz_remain": -141.0}) + return {"status": "ok"} + + sampler = FakeSampler() + result = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": 1000}, + verbose=True, + stream=stream, + interval_seconds=0.0, + ) + + assert result == {"status": "ok"} + assert sampler.kwargs["show_status"] is False + assert callable(sampler.kwargs["viz_callback"]) + output = stream.getvalue() + assert "Using simple progress updates" in output + assert "[ultranest] running" in output + assert "[ultranest] done 100.00%" in output + + +def test_run_reactive_sampler_silent_mode(monkeypatch): + _reset_ultranest_env(monkeypatch) + stream = _FakeStream(tty=False) + + class FakeSampler: + def __init__(self): + self.kwargs = None + + def run(self, **kwargs): + self.kwargs = kwargs + return {"status": "ok"} + + sampler = FakeSampler() + result = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": 1000}, + verbose=False, + stream=stream, + ) + + assert result == {"status": "ok"} + assert sampler.kwargs["show_status"] is False + assert sampler.kwargs["viz_callback"] is False + assert stream.getvalue() == "" + + +def test_run_reactive_sampler_translates_ultranest_degenerate_region_value_error(monkeypatch): + _reset_ultranest_env(monkeypatch) + + class FakeSampler: + def run(self, **kwargs): + exec( + compile( + 'raise ValueError("Buffer has wrong number of dimensions (expected 2, got 0)")', + "ultranest/mlfriends.pyx", + "exec", + ), + {}, + ) + + with pytest.raises(np.linalg.LinAlgError) as excinfo: + run_reactive_sampler(FakeSampler(), verbose=False) + + assert "degenerate sampling region" in str(excinfo.value) + assert isinstance(excinfo.value.__cause__, ValueError) + + +def test_run_reactive_sampler_translates_ultranest_bounding_ellipsoid_assertion(monkeypatch): + _reset_ultranest_env(monkeypatch) + + class FakeSampler: + def run(self, **kwargs): + exec( + compile( + "def bounding_ellipsoid():\n" + " raise AssertionError('(array(nan), array([[0.58720908]]))')\n" + "bounding_ellipsoid()", + "ultranest/mlfriends.pyx", + "exec", + ), + {}, + ) + + with pytest.raises(np.linalg.LinAlgError) as excinfo: + run_reactive_sampler(FakeSampler(), verbose=False) + + assert "degenerate sampling region" in str(excinfo.value) + assert isinstance(excinfo.value.__cause__, AssertionError) + + +def test_run_reactive_sampler_preserves_other_ultranest_mlfriends_assertion(monkeypatch): + _reset_ultranest_env(monkeypatch) + + class FakeSampler: + def run(self, **kwargs): + exec( + compile( + "def compute_enlargement():\n" + " raise AssertionError('not a bounding ellipsoid failure')\n" + "compute_enlargement()", + "ultranest/mlfriends.pyx", + "exec", + ), + {}, + ) + + with pytest.raises(AssertionError, match="not a bounding ellipsoid failure"): + run_reactive_sampler(FakeSampler(), verbose=False) + + +def test_run_reactive_sampler_preserves_unrelated_value_error(monkeypatch): + _reset_ultranest_env(monkeypatch) + + class FakeSampler: + def run(self, **kwargs): + raise ValueError("not an ultranest region error") + + with pytest.raises(ValueError, match="not an ultranest region error"): + run_reactive_sampler(FakeSampler(), verbose=False) + + +def test_run_reactive_sampler_applies_fast_defaults(monkeypatch): + _reset_ultranest_env(monkeypatch) + + class FakeSampler: + def __init__(self): + self.kwargs = None + + def run(self, **kwargs): + self.kwargs = kwargs + return {"status": "ok"} + + sampler = FakeSampler() + run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": 1000}, + verbose=False, + ) + + assert sampler.kwargs["min_num_live_points"] == 200 + assert sampler.kwargs["min_ess"] == 200 + assert sampler.kwargs["dlogz"] == 1.0 + assert sampler.kwargs["dKL"] == 1.0 + assert sampler.kwargs["frac_remain"] == 0.05 + assert sampler.kwargs["max_num_improvement_loops"] == 1 + assert sampler.kwargs["max_ncalls"] == 1000 + + +def test_run_reactive_sampler_uses_env_live_point_override(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_MIN_NUM_LIVE_POINTS", "320") + + class FakeSampler: + def __init__(self): + self.kwargs = None + + def run(self, **kwargs): + self.kwargs = kwargs + return {"status": "ok"} + + sampler = FakeSampler() + run_reactive_sampler(sampler, verbose=False) + + assert sampler.kwargs["min_num_live_points"] == 320 + + +def test_run_reactive_sampler_auto_scales_draw_size_by_workers_and_ram(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_WORKERS", "72") + monkeypatch.setenv("EXOTIC_ULTRANEST_WORKER_BACKEND", "thread") + monkeypatch.setattr(ultranest_utils, "_available_cpu_count", lambda: 72) + monkeypatch.setattr(ultranest_utils, "_system_total_memory_bytes", lambda: 128 * 1024 ** 3) + + class FakeSampler: + def __init__(self): + self.ndraw_min = 128 + self.ndraw_max = 65536 + self.draw_multiple = True + self.x_dim = 6 + self.num_params = 6 + self.loglike = lambda params: np.zeros(np.asarray(params).shape[0]) + self.kwargs = None + + def run(self, **kwargs): + self.kwargs = kwargs + return {"status": "ok"} + + sampler = FakeSampler() + run_reactive_sampler(sampler, verbose=False) + + assert sampler.ndraw_min == ( + 72 + * ultranest_utils.HIGH_AUTO_POINTS_PER_WORKER + * ultranest_utils.AUTO_POINTS_PER_WORKER_MULTIPLIER + ) + assert sampler.ndraw_max == 65536 + + +def test_run_reactive_sampler_auto_uses_smaller_chunks_when_ram_per_cpu_is_low(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_WORKERS", "72") + monkeypatch.setenv("EXOTIC_ULTRANEST_WORKER_BACKEND", "thread") + monkeypatch.setattr(ultranest_utils, "_available_cpu_count", lambda: 72) + monkeypatch.setattr(ultranest_utils, "_system_total_memory_bytes", lambda: 16 * 1024 ** 3) + + class FakeSampler: + def __init__(self): + self.ndraw_min = 128 + self.ndraw_max = 65536 + self.draw_multiple = True + self.x_dim = 6 + self.num_params = 6 + self.loglike = lambda params: np.zeros(np.asarray(params).shape[0]) + self.kwargs = None + + def run(self, **kwargs): + self.kwargs = kwargs + return {"status": "ok"} + + sampler = FakeSampler() + run_reactive_sampler(sampler, verbose=False) + + assert sampler.ndraw_min == ( + 72 + * ultranest_utils.MIN_AUTO_POINTS_PER_WORKER + * ultranest_utils.AUTO_POINTS_PER_WORKER_MULTIPLIER + ) + + +def test_configured_ultranest_workers_defaults_to_available_cpu_count(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setattr(ultranest_utils.os, "process_cpu_count", lambda: 12, raising=False) + + assert ultranest_utils._configured_ultranest_workers() == 12 + + +def test_configured_ultranest_workers_accepts_auto_override(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_WORKERS", "auto") + monkeypatch.setattr(ultranest_utils.os, "process_cpu_count", lambda: 10, raising=False) + + assert ultranest_utils._configured_ultranest_workers() == 10 + + +def test_configured_ultranest_workers_preserves_numeric_override(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_WORKERS", "3") + monkeypatch.setattr(ultranest_utils.os, "process_cpu_count", lambda: 12, raising=False) + + assert ultranest_utils._configured_ultranest_workers() == 3 + + +def test_run_reactive_sampler_parallelizes_vectorized_loglike_batches(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_WORKERS", "3") + monkeypatch.setenv("EXOTIC_ULTRANEST_WORKER_BACKEND", "thread") + + class FakeSampler: + def __init__(self): + self.kwargs = None + self.chunk_sizes = [] + + def loglike(points): + self.chunk_sizes.append(len(points)) + return points[:, 0] + + self.loglike = loglike + + def run(self, **kwargs): + self.kwargs = kwargs + values = self.loglike(np.arange(12, dtype=float).reshape(6, 2)) + return {"values": values} + + sampler = FakeSampler() + result = run_reactive_sampler(sampler, verbose=False) + + assert result["values"].tolist() == [0, 2, 4, 6, 8, 10] + assert sorted(sampler.chunk_sizes) == [2, 2, 2] + + +def test_run_reactive_sampler_auto_workers_uses_available_cpu_count(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_WORKER_BACKEND", "thread") + monkeypatch.setattr(ultranest_utils.os, "process_cpu_count", lambda: 4, raising=False) + + class FakeSampler: + def __init__(self): + self.chunk_sizes = [] + + def loglike(points): + self.chunk_sizes.append(len(points)) + return points[:, 0] + + self.loglike = loglike + + def run(self, **kwargs): + return {"values": self.loglike(np.arange(16, dtype=float).reshape(8, 2))} + + sampler = FakeSampler() + result = run_reactive_sampler(sampler, verbose=False) + + assert result["values"].tolist() == [0, 2, 4, 6, 8, 10, 12, 14] + assert sorted(sampler.chunk_sizes) == [2, 2, 2, 2] + + +def test_process_backend_disables_parent_gc_while_pool_is_active(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_WORKERS", "2") + monkeypatch.setenv("EXOTIC_ULTRANEST_WORKER_BACKEND", "process") + monkeypatch.setattr(ultranest_utils.sys, "platform", "linux") + monkeypatch.setattr(ultranest_utils, "_is_colab_runtime", lambda: False) + monkeypatch.setattr( + ultranest_utils, + "get_mpi_status", + lambda: {"available": False, "size": 1, "rank": 0, "source": "test", "error": None}, + ) + + cleanup_calls = [] + monkeypatch.setattr( + ultranest_utils, + "suppress_inherited_tk_cleanup_in_worker", + lambda: cleanup_calls.append("suppress"), + ) + + pool_events = [] + + class FakePool: + def __init__(self, processes, initializer): + pool_events.append(("init", processes, initializer)) + + def map(self, func, chunks): + pool_events.append(("map", ultranest_utils.gc.isenabled())) + return [func(chunk) for chunk in chunks] + + def close(self): + pool_events.append(("close", None)) + + def join(self): + pool_events.append(("join", None)) + + class FakeContext: + Pool = FakePool + + monkeypatch.setattr(ultranest_utils.multiprocessing, "get_context", lambda _method: FakeContext()) + + class FakeSampler: + def __init__(self): + self.chunk_sizes = [] + + def loglike(points): + self.chunk_sizes.append(len(points)) + return points[:, 0] + + self.loglike = loglike + + def run(self, **kwargs): + assert ultranest_utils.gc.isenabled() is False + return {"values": self.loglike(np.arange(8, dtype=float).reshape(4, 2))} + + gc_was_enabled = ultranest_utils.gc.isenabled() + ultranest_utils.gc.enable() + try: + sampler = FakeSampler() + result = run_reactive_sampler(sampler, verbose=False) + assert ultranest_utils.gc.isenabled() is True + finally: + if not gc_was_enabled: + ultranest_utils.gc.disable() + + assert result["values"].tolist() == [0, 2, 4, 6] + assert sorted(sampler.chunk_sizes) == [2, 2] + assert cleanup_calls == ["suppress"] + assert pool_events[0] == ("init", 2, ultranest_utils.suppress_inherited_tk_cleanup_in_worker) + assert ("map", False) in pool_events + assert pool_events[-2:] == [("close", None), ("join", None)] + + +def test_process_worker_initializer_suppresses_inherited_tk_destructors(monkeypatch): + class FakeImage: + def __del__(self): + raise RuntimeError("main thread is not in main loop") + + class FakeVariable: + def __del__(self): + raise RuntimeError("main thread is not in main loop") + + original_image_del = FakeImage.__del__ + original_variable_del = FakeVariable.__del__ + fake_tkinter = types.SimpleNamespace(Image=FakeImage, Variable=FakeVariable) + monkeypatch.setitem(sys.modules, "tkinter", fake_tkinter) + + assert ultranest_utils._suppress_inherited_tk_cleanup() is True + + assert FakeImage._exotic_worker_original_del is original_image_del + assert FakeVariable._exotic_worker_original_del is original_variable_del + assert FakeImage().__del__() is None + assert FakeVariable().__del__() is None + assert ultranest_utils._suppress_inherited_tk_cleanup() is False + + +def test_run_reactive_sampler_preserves_explicit_live_point_override(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_MIN_NUM_LIVE_POINTS", "320") + + class FakeSampler: + def __init__(self): + self.kwargs = None + + def run(self, **kwargs): + self.kwargs = kwargs + return {"status": "ok"} + + sampler = FakeSampler() + run_reactive_sampler( + sampler, + run_kwargs={"min_num_live_points": 450}, + verbose=False, + ) + + assert sampler.kwargs["min_num_live_points"] == 450 + + +def test_run_reactive_sampler_silences_mpi_worker_rank(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setattr( + ultranest_utils, + "get_mpi_status", + lambda: { + "available": True, + "size": 4, + "rank": 2, + "source": "mpi4py", + "error": None, + }, + ) + stream = _FakeStream(tty=True) + + class FakeSampler: + def __init__(self): + self.kwargs = None + + def run(self, **kwargs): + self.kwargs = kwargs + return {"status": "ok"} + + sampler = FakeSampler() + result = run_reactive_sampler( + sampler, + verbose=True, + stream=stream, + ) + + assert result == {"status": "ok"} + assert sampler.kwargs["show_status"] is False + assert sampler.kwargs["viz_callback"] is False + assert stream.getvalue() == "" + + +def test_run_reactive_sampler_tty_defaults_to_simple_status(monkeypatch): + _reset_ultranest_env(monkeypatch) + stream = _FakeStream(tty=True) + + class FakeSampler: + def __init__(self): + self.kwargs = None + + def run(self, **kwargs): + self.kwargs = kwargs + return {"status": "ok"} + + sampler = FakeSampler() + result = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": 1000}, + verbose=True, + stream=stream, + ) + + assert result == {"status": "ok"} + assert sampler.kwargs["show_status"] is False + assert callable(sampler.kwargs["viz_callback"]) + output = stream.getvalue() + assert "Using simple progress updates" in output + assert "10s heartbeat" in output + + +def test_run_reactive_sampler_plain_override_uses_simple_status(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_PLAIN_PROGRESS", "1") + stream = _FakeStream(tty=True) + + class FakeSampler: + def __init__(self): + self.kwargs = None + + def run(self, **kwargs): + self.kwargs = kwargs + return {"status": "ok"} + + sampler = FakeSampler() + result = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": 1000}, + verbose=True, + stream=stream, + ) + + assert result == {"status": "ok"} + assert sampler.kwargs["show_status"] is False + assert callable(sampler.kwargs["viz_callback"]) + output = stream.getvalue() + assert "Using simple progress updates" in output + assert "10s heartbeat" in output + + +def test_run_reactive_sampler_rich_override_uses_native_status(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_RICH_PROGRESS", "1") + monkeypatch.setenv("TERM", "xterm-256color") + stream = _FakeStream(tty=True) + + class FakeSampler: + def __init__(self): + self.kwargs = None + + def run(self, **kwargs): + self.kwargs = kwargs + return {"status": "ok"} + + sampler = FakeSampler() + result = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": 1000}, + verbose=True, + stream=stream, + ) + + assert result == {"status": "ok"} + assert sampler.kwargs["show_status"] is True + assert "viz_callback" not in sampler.kwargs + assert stream.getvalue() == "" + + +def test_run_reactive_sampler_silent_mode_mutes_ultranest_logger(monkeypatch): + _reset_ultranest_env(monkeypatch) + progress_stream = _FakeStream(tty=False) + log_stream = io.StringIO() + + class FakeSampler: + def __init__(self): + self.kwargs = None + self.logger = logging.getLogger("ultranest.tests.silent_mode") + self.logger.handlers = [] + self.logger.propagate = False + handler = logging.StreamHandler(log_stream) + handler.setLevel(logging.INFO) + self.logger.addHandler(handler) + self.logger.setLevel(logging.INFO) + + def run(self, **kwargs): + self.kwargs = kwargs + self.logger.info("native ultranest noise") + return {"status": "ok"} + + sampler = FakeSampler() + result = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": 1000}, + verbose=False, + stream=progress_stream, + ) + + assert result == {"status": "ok"} + assert sampler.kwargs["show_status"] is False + assert sampler.kwargs["viz_callback"] is False + assert progress_stream.getvalue() == "" + assert log_stream.getvalue() == "" + + +def test_run_reactive_sampler_plain_mode_mutes_ultranest_logger(monkeypatch): + _reset_ultranest_env(monkeypatch) + monkeypatch.setenv("EXOTIC_ULTRANEST_PLAIN_PROGRESS", "1") + progress_stream = _FakeStream(tty=False) + log_stream = io.StringIO() + + class FakeSampler: + def __init__(self): + self.kwargs = None + self.logger = logging.getLogger("ultranest.tests.plain_mode") + self.logger.handlers = [] + self.logger.propagate = False + handler = logging.StreamHandler(log_stream) + handler.setLevel(logging.INFO) + self.logger.addHandler(handler) + self.logger.setLevel(logging.INFO) + + def run(self, **kwargs): + self.kwargs = kwargs + callback = kwargs["viz_callback"] + self.logger.info("native ultranest noise") + callback(None, {"it": 25, "ncall": 40, "logz": -10.0, "logz_remain": -9.5}) + return {"status": "ok"} + + sampler = FakeSampler() + result = run_reactive_sampler( + sampler, + run_kwargs={"max_ncalls": 1000}, + verbose=True, + stream=progress_stream, + interval_seconds=0.0, + ) + + assert result == {"status": "ok"} + assert sampler.kwargs["show_status"] is False + assert callable(sampler.kwargs["viz_callback"]) + assert "Using simple progress updates" in progress_stream.getvalue() + assert "native ultranest noise" not in progress_stream.getvalue() + assert log_stream.getvalue() == "" diff --git a/tests/test_utils.py b/tests/test_utils.py index 9049ecdc..4f748032 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,70 @@ from exotic.utils import * from unittest.mock import patch +import pytest + + +def test_coerce_boolean_config_value_accepts_all_supported_forms(): + for value in (True, 1, "1", "y", "Y", "yes", "TRUE", "on"): + assert coerce_boolean_config_value(value) is True + + for value in (False, 0, "0", "n", "N", "no", "FALSE", "off"): + assert coerce_boolean_config_value(value) is False + + +def test_coerce_boolean_config_value_rejects_non_boolean_values(): + for value in (None, 2, -1, "sometimes", [], {}): + assert coerce_boolean_config_value(value) is None + + +def test_filename_date_token_uses_date_only_for_iso_timestamp(): + assert filename_date_token("2026-05-06T19:51:13.964-0700") == "2026-05-06" + assert filename_date_token("20260506T195113") == "2026-05-06" + assert filename_date_token("2026/05/06 19:51:13") == "2026-05-06" + + +def test_safe_output_filename_sanitizes_filename_chars(): + filename = safe_output_filename( + "BestFit", + "XO-1/b ", + filename_date_token("2026-05-06T19:51:13.964-0700"), + extension=" png", + ) + + assert filename == "BestFit_XO-1-b_2026-05-06.png" + + +def test_safe_output_filename_removes_spaces_from_planet_names(): + filename = safe_output_filename( + "FinalLightCurve", + "Kepler-12 b", + "03-JUN-2026", + extension="png", + ) + + assert filename == "FinalLightCurve_Kepler-12b_03-JUN-2026.png" + assert " " not in filename + + +@pytest.mark.parametrize( + ("planet_name", "expected"), + ( + ("TOI-4010b", "TOI-4010 b"), + ("Kepler-11c", "Kepler-11 c"), + ("HD 41004Ag", "HD 41004A g"), + ("TOI-4010 b", "TOI-4010 b"), + ("Candidate", "Candidate"), + ), +) +def test_format_aavso_exoplanet_name_separates_planet_suffix(planet_name, expected): + assert format_aavso_exoplanet_name(planet_name) == expected + + +def test_sanitize_filename_component_cleans_fallback(): + filename = sanitize_filename_component(" ", fallback="bad fallback") + + assert filename == "badfallback" + class TestUserInput: """tests the `user_input()` function""" @@ -346,6 +410,23 @@ def test_small_numbers(self): assert 0.0002 == result +class TestFormatValueAndUncertainty: + def test_preserves_two_significant_figures_and_matches_value_precision(self): + assert format_value_and_uncertainty(0.073, 0.01) == ("0.073", "0.010") + assert format_value_and_uncertainty(1.0, 0.00023) == ("1.00000", "0.00023") + assert format_value_and_uncertainty(0.0, 0.0031) == ("0.0000", "0.0031") + + def test_formats_uncertainties_above_one_to_two_significant_figures(self): + assert format_value_and_uncertainty(89.3511, 2.16) == ("89.4", "2.2") + assert format_value_and_uncertainty(1234, 100) == ("1230", "1.0e+02") + + def test_recomputes_precision_when_rounding_crosses_a_decade(self): + assert format_value_and_uncertainty(0.0732, 0.00999) == ("0.073", "0.010") + + def test_full_report_text_uses_the_same_precision(self): + assert format_value_with_uncertainty(12.0, 0.4) == "12.00 +/- 0.40" + + class TestGetVal: """tests the get_val() function @@ -443,6 +524,23 @@ def test_process_lat_long_dms_inputs(self): assert self._EXPECTED_LONGITUDE_RESULT == process_lat_long("+152:30:36", "longitude") assert self._EXPECTED_LATITUDE_RESULT == process_lat_long("+37:2:24", "latitude") + @pytest.mark.parametrize( + ("value", "coordinate_type", "expected"), + ( + ("28 17 58.8 N", "latitude", 28.2996666667), + ("28 17 58.8 S", "latitude", -28.2996666667), + ("16 30 39.7 E", "longitude", 16.5110277778), + ("16 30 39.7 W", "longitude", -16.5110277778), + ("-16 30 39.7 W", "longitude", -16.5110277778), + ("S28:17:58.8", "latitude", -28.2996666667), + ), + ) + def test_process_lat_long_hemisphere_inputs(self, value, coordinate_type, expected): + assert float(process_lat_long(value, coordinate_type)) == pytest.approx(expected) + + def test_process_lat_long_rejects_wrong_hemisphere_for_axis(self): + assert process_lat_long("28 17 58.8 W", "latitude") is None + @patch("builtins.print") def test_bad_inputs(self, mock_print): result = process_lat_long("foo", "longitude") @@ -567,6 +665,18 @@ def test_generic_hdr(self, mock_pll): assert result == hdr["LAT"] # NOTE: actual return value is "+34.560000" but I mocked this call + def test_generic_hdr_interprets_coordinate_hemispheres(self): + hdr = { + "SITELAT": "28 17 58.8 S", + "SITELONG": "16 30 39.7 W", + } + + latitude_result = find(hdr, ['LATITUDE', 'LAT', 'SITELAT']) + longitude_result = find(hdr, ['LONGITUD', 'LONG', 'LONGITUDE', 'SITELONG']) + + assert float(latitude_result) == pytest.approx(-28.2996666667) + assert float(longitude_result) == pytest.approx(-16.5110277778) + @patch("exotic.utils.get_val") def test_ks_zero_not_expected(self, mock_get_val): # NOTE: returns whatever is returned in `val = get_val()`